You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/12 11:48:33 UTC
[01/48] kylin git commit: minor,
add kylin.properties for Spark on HDP [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-hbase0.98 c4fb2977e -> 1877487fa (forced update)
minor, add kylin.properties for Spark on HDP
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc1bac9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc1bac9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc1bac9f
Branch: refs/heads/master-hbase0.98
Commit: cc1bac9f452fddcfd316a23c4d21bfb7e4261ff0
Parents: 9ff78bc
Author: Li Yang <li...@apache.org>
Authored: Fri Feb 24 17:20:02 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Feb 24 17:20:02 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 5 +++++
.../apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc1bac9f/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 46fe86b..cfceef4 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -230,3 +230,8 @@ kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-histo
## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
+
+## uncomment for HDP
+#kylin.engine.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
+#kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+#kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc1bac9f/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index f68194e..e0b3e6c 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -78,7 +78,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
try {
return ClassUtil.findContainingJar(Class.forName(className), perferLibraryName);
} catch (ClassNotFoundException e) {
- logger.warn("failed to locate jar for class " + className + ", ignore it", e);
+ logger.warn("failed to locate jar for class " + className + ", ignore it");
}
return "";
[33/48] kylin git commit: KYLIN-2222 web ui uses rest api to decide
which dim encoding is valid for different typed columns
Posted by li...@apache.org.
KYLIN-2222 web ui uses rest api to decide which dim encoding is valid for different typed columns
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8edf234c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8edf234c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8edf234c
Branch: refs/heads/master-hbase0.98
Commit: 8edf234ce571593145970e4d6cc18934cf761348
Parents: c5f3b22
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 10 15:58:00 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 17:17:11 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/EncodingService.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8edf234c/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
index 7d7d016..60f9974 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.kylin.rest.service;
import java.util.List;
@@ -44,8 +43,8 @@ public class EncodingService extends BasicService {
} else if (dataType.isDateTimeFamily()) {
return Lists.newArrayList(DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME);
} else if (dataType.isStringFamily()) {
- return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, //
- FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME);
+ return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, //
+ FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME);
} else {
throw new IllegalArgumentException("can't provide valid encodings for datatype:" + dataType);
}
[22/48] kylin git commit: minor changes on IT ports to allow
concurrent CI
Posted by li...@apache.org.
minor changes on IT ports to allow concurrent CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a21ffc83
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a21ffc83
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a21ffc83
Branch: refs/heads/master-hbase0.98
Commit: a21ffc839d46248f219cf81c0412b416059f6fa8
Parents: cde40f3
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Mar 7 16:59:20 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 7 16:59:20 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 11 ++++++++---
.../apache/kylin/restclient/ITRestClientTest.java | 18 ++++++++++--------
2 files changed, 18 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a21ffc83/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 05f615f..bdf61f8 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -28,6 +28,7 @@ import java.sql.Statement;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
+import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -37,6 +38,8 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
@@ -46,9 +49,12 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
private static Server server = null;
private static SystemPropertiesOverride sysPropsOverride = new SystemPropertiesOverride();
+ private static final int PORT = new Random().nextInt(100) + 37070;
+ private static final Logger logger = LoggerFactory.getLogger(ITJDBCDriverTest.class);
@BeforeClass
public static void beforeClass() throws Exception {
+ logger.info("random jetty port: " + PORT);
sysPropsOverride.override("spring.profiles.active", "testing");
sysPropsOverride.override("catalina.home", "."); // resources/log4j.properties ref ${catalina.home}
staticCreateTestMetadata();
@@ -74,7 +80,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
protected static void startJetty() throws Exception {
- server = new Server(7070);
+ server = new Server(PORT);
WebAppContext context = new WebAppContext();
context.setDescriptor("../server/src/main/webapp/WEB-INF/web.xml");
@@ -94,7 +100,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
Properties info = new Properties();
info.put("user", "ADMIN");
info.put("password", "KYLIN");
- Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
+ Connection conn = driver.connect("jdbc:kylin://localhost:" + PORT + "/default", info);
return conn;
}
@@ -262,7 +268,6 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
}
-
@Test
public void testResultSetWithMaxRows() throws Exception {
String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME ";
http://git-wip-us.apache.org/repos/asf/kylin/blob/a21ffc83/kylin-it/src/test/java/org/apache/kylin/restclient/ITRestClientTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/restclient/ITRestClientTest.java b/kylin-it/src/test/java/org/apache/kylin/restclient/ITRestClientTest.java
index 2afd5c9..c97f08b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/restclient/ITRestClientTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/restclient/ITRestClientTest.java
@@ -18,6 +18,13 @@
package org.apache.kylin.restclient;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Random;
+
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpResponse;
import org.apache.kylin.common.restclient.RestClient;
@@ -30,12 +37,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.util.HashMap;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
/**
*/
public class ITRestClientTest extends HBaseMetadataTestCase {
@@ -46,7 +47,7 @@ public class ITRestClientTest extends HBaseMetadataTestCase {
private static final String HOST = "localhost";
- private static final int PORT = 7070;
+ private static final int PORT = new Random().nextInt(100) + 37070;
private static final String USERNAME = "ADMIN";
@@ -60,6 +61,7 @@ public class ITRestClientTest extends HBaseMetadataTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
+ logger.info("random jetty port: " + PORT);
sysPropsOverride.override("spring.profiles.active", "testing");
sysPropsOverride.override("catalina.home", "."); // resources/log4j.properties ref ${catalina.home}
staticCreateTestMetadata();
@@ -113,7 +115,7 @@ public class ITRestClientTest extends HBaseMetadataTestCase {
protected static void startJetty() throws Exception {
- server = new Server(7070);
+ server = new Server(PORT);
WebAppContext context = new WebAppContext();
context.setDescriptor("../server/src/main/webapp/WEB-INF/web.xml");
[26/48] kylin git commit: KYLIN-2477 Make UT and IT compatible with
any kinds of metadata resource store
Posted by li...@apache.org.
KYLIN-2477 Make UT and IT compatible with any kinds of metadata resource store
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01abb00e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01abb00e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01abb00e
Branch: refs/heads/master-hbase0.98
Commit: 01abb00e45047c06d72d47d1eead136d2fa87d3d
Parents: 4f7885c
Author: xiefan46 <95...@qq.com>
Authored: Thu Mar 2 17:49:45 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 8 16:53:09 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 1 +
.../apache/kylin/common/KylinConfigBase.java | 19 ++++++++++-
.../persistence/LocalFileResourceStoreTest.java | 3 +-
.../common/persistence/ResourceStoreTest.java | 36 ++++++++++++++++++--
.../storage/hbase/ITHBaseResourceStoreTest.java | 12 ++++---
.../storage/hdfs/ITHDFSResourceStoreTest.java | 33 +++++++-----------
.../kylin/storage/hdfs/ITLockManagerTest.java | 4 +--
.../kylin/storage/hdfs/HDFSResourceStore.java | 34 ++----------------
.../apache/kylin/storage/hdfs/LockManager.java | 7 ++--
.../apache/kylin/storage/hdfs/ResourceLock.java | 7 ++--
10 files changed, 87 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index cfceef4..1a55c94 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -235,3 +235,4 @@ kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-histo
#kylin.engine.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
#kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
#kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5d665df..917879d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -954,8 +954,25 @@ abstract public class KylinConfigBase implements Serializable {
}
//ResourceStore Impl
- public String getResourceStoreImpl() {
+ /*public String getResourceStoreImpl() {
return getOptional("kylin.metadata.default-resource-store-impl", "org.apache.kylin.storage.hbase.HBaseResourceStore");
+ }*/
+
+ public Map<String, String> getResourceStoreImpls() {
+ Map<String, String> r = Maps.newLinkedHashMap();
+ // ref constants in ISourceAware
+ r.put("", "org.apache.kylin.common.persistence.FileResourceStore");
+ r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore");
+ r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore");
+ r.putAll(getPropertiesByPrefix("kylin.resource.store.provider."));
+ return r;
+ }
+
+ public String getResourceStoreImpl() {
+ String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
+ int cut = metadataUrl.indexOf('@');
+ String key = cut < 0 ? "" : metadataUrl.substring(0, cut);
+ return getResourceStoreImpls().get(key);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
index cc6143d..db22d96 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
@@ -38,8 +38,7 @@ public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
@Test
public void testFileStore() throws Exception {
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- ResourceStoreTest.testAStore(store);
+ ResourceStoreTest.testAStore("org.apache.kylin.common.persistence.FileResourceStore", "", KylinConfig.getInstanceFromEnv());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 79a0c30..10714a5 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -30,6 +30,8 @@ import java.util.List;
import java.util.NavigableSet;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,14 +44,30 @@ public class ResourceStoreTest {
private static final String PERFORMANCE_TEST_ROOT_PATH = "/performance";
- private static final int TEST_RESOURCE_COUNT = 1000;
+ private static final int TEST_RESOURCE_COUNT = 100;
- public static void testAStore(ResourceStore store) throws IOException {
+ public static void testAStore(String storeName, String url, KylinConfig kylinConfig) throws Exception {
+ String oldUrl = replaceMetadataUrl(kylinConfig, url);
+ testAStore(getStoreByName(storeName, kylinConfig));
+ replaceMetadataUrl(kylinConfig, oldUrl);
+ }
+
+ public static void testPerformance(String storeName, String url, KylinConfig kylinConfig) throws Exception {
+ String oldUrl = replaceMetadataUrl(kylinConfig, url);
+ testPerformance(getStoreByName(storeName, kylinConfig));
+ replaceMetadataUrl(kylinConfig, oldUrl);
+ }
+
+ public static String mockUrl(String tag, KylinConfig kylinConfig) {
+ return kylinConfig.getMetadataUrlPrefix() + "@" + tag;
+ }
+
+ private static void testAStore(ResourceStore store) throws IOException {
testBasics(store);
testGetAllResources(store);
}
- public static void testPerformance(ResourceStore store) throws IOException {
+ private static void testPerformance(ResourceStore store) throws IOException {
logger.info("Test basic functions");
testAStore(store);
logger.info("Basic function ok. Start to test performance for class : " + store.getClass());
@@ -229,4 +247,16 @@ public class ResourceStoreTest {
}
}
+ public static String replaceMetadataUrl(KylinConfig kylinConfig, String newUrl) {
+ String oldUrl = kylinConfig.getMetadataUrl();
+ kylinConfig.setProperty("kylin.metadata.url", newUrl);
+ return oldUrl;
+ }
+
+ private static ResourceStore getStoreByName(String storeName, KylinConfig kylinConfig) throws Exception {
+ Class<? extends ResourceStore> cls = ClassUtil.forName(storeName, ResourceStore.class);
+ ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+ return store;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
index 618488f..4e35118 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceStoreTest;
import org.apache.kylin.common.persistence.ResourceStoreTest.StringEntity;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -36,9 +35,12 @@ import org.junit.Test;
public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
+ private KylinConfig kylinConfig;
+
@Before
public void setup() throws Exception {
this.createTestMetadata();
+ kylinConfig = KylinConfig.getInstanceFromEnv();
}
@After
@@ -48,8 +50,8 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
@Test
public void testHBaseStore() throws Exception {
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- ResourceStoreTest.testAStore(store);
+ String storeName = "org.apache.kylin.storage.hbase.HBaseResourceStore";
+ ResourceStoreTest.testAStore(storeName, ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig);
}
@Test
@@ -57,7 +59,8 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
String path = "/cube/_test_large_cell.json";
String largeContent = "THIS_IS_A_LARGE_CELL";
StringEntity content = new StringEntity(largeContent);
- HBaseResourceStore store = (HBaseResourceStore) ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ String oldUrl = ResourceStoreTest.replaceMetadataUrl(kylinConfig, ResourceStoreTest.mockUrl("hbase", kylinConfig));
+ HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv());
Configuration hconf = store.getConnection().getConfiguration();
int origSize = Integer.parseInt(hconf.get("hbase.client.keyvalue.maxsize", "10485760"));
@@ -81,6 +84,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
in.close();
store.deleteResource(path);
+ ResourceStoreTest.replaceMetadataUrl(kylinConfig, oldUrl);
} finally {
hconf.set("hbase.client.keyvalue.maxsize", "" + origSize);
store.deleteResource(path);
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 534839f..0a95f1c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStoreTest;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.storage.hbase.HBaseResourceStore;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -35,9 +34,11 @@ import org.junit.Test;
public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
- KylinConfig kylinConfig;
- FileSystem fs;
- String workingDir;
+ private KylinConfig kylinConfig;
+
+ private FileSystem fs;
+
+ private String workingDir;
@Before
public void setup() throws Exception {
@@ -78,29 +79,21 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
}
private void doTestWithPath(String path) throws Exception {
- String oldUrl = kylinConfig.getMetadataUrl();
- kylinConfig.setProperty("kylin.metadata.url", path + "@hdfs");
- HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
- ResourceStoreTest.testAStore(store);
- kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+ String storeName = "org.apache.kylin.storage.hdfs.HDFSResourceStore";
+ ResourceStoreTest.testAStore(storeName, ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig);
assertTrue(fs.exists(new Path(path)));
}
@Ignore
@Test
- public void performanceTest() throws Exception{
+ public void performanceTest() throws Exception {
+
//test hdfs performance
- String oldUrl = kylinConfig.getMetadataUrl();
- kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hdfs");
- HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
- ResourceStoreTest.testPerformance(store);
- kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+ String hdfsStoreName = "org.apache.kylin.storage.hdfs.HDFSResourceStore";
+ ResourceStoreTest.testPerformance(hdfsStoreName, ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig);
//test hbase
- oldUrl = kylinConfig.getMetadataUrl();
- kylinConfig.setProperty("kylin.metadata.url", "kylin_metadata@hbase");
- HBaseResourceStore store2 = new HBaseResourceStore(kylinConfig);
- ResourceStoreTest.testPerformance(store2);
- kylinConfig.setProperty("kylin.metadata.url", oldUrl);
+ String hbaseStoreName = "org.apache.kylin.storage.hbase.HBaseResourceStore.HBaseResourceStore";
+ ResourceStoreTest.testPerformance(hbaseStoreName, ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index a6cf43b..3449f87 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -193,9 +193,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
public void doWork(long time, TimeUnit unit) throws Exception {
ResourceLock lock = lockManager.getLock(lockPath);
- if (!lock.acquire(time, unit)) {
- throw new IllegalStateException(clientName + " could not acquire the lock");
- }
+ lock.acquire(time, unit);
try {
logger.info(clientName + " has the lock");
resource.use();
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index d24d3b4..6744805 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.TreeSet;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -46,14 +45,10 @@ public class HDFSResourceStore extends ResourceStore {
private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class);
- private static final long DEFAULT_ACQUIRE_LOCK_TIMEOUT = 2;
-
private Path hdfsMetaPath;
private FileSystem fs;
- private LockManager lockManager;
-
public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
super(kylinConfig);
String metadataUrl = kylinConfig.getMetadataUrl();
@@ -68,8 +63,6 @@ public class HDFSResourceStore extends ResourceStore {
String path = metadataUrl.substring(0, cut);
fs = HadoopUtil.getFileSystem(path);
Path metadataPath = new Path(path);
- //creat lock manager
- this.lockManager = new LockManager(kylinConfig, getRelativePath(metadataPath));
if (fs.exists(metadataPath) == false) {
logger.warn("Path not exist in HDFS, create it: " + path);
createMetaFolder(metadataPath, kylinConfig);
@@ -80,20 +73,12 @@ public class HDFSResourceStore extends ResourceStore {
}
-
-
private void createMetaFolder(Path metaDirName, KylinConfig kylinConfig) throws Exception {
//create hdfs meta path
- ResourceLock lock = lockManager.getLock(getRelativePath(metaDirName));
- try {
- if (lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS)) {
- if (!fs.exists(metaDirName)) {
- fs.mkdirs(metaDirName);
- }
- }
- } finally {
- lockManager.releaseLock(lock);
+ if (!fs.exists(metaDirName)) {
+ fs.mkdirs(metaDirName);
}
+
logger.info("hdfs meta path created: " + metaDirName.toString());
}
@@ -164,10 +149,7 @@ public class HDFSResourceStore extends ResourceStore {
return 0;
}
FSDataInputStream in = null;
- ResourceLock lock = null;
try {
- lock = lockManager.getLock(resPath);
- lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS);
in = fs.open(p);
long t = in.readLong();
return t;
@@ -175,7 +157,6 @@ public class HDFSResourceStore extends ResourceStore {
throw new IOException("Put resource fail", e);
} finally {
IOUtils.closeQuietly(in);
- lockManager.releaseLock(lock);
}
}
@@ -186,10 +167,7 @@ public class HDFSResourceStore extends ResourceStore {
Path p = getRealHDFSPath(resPath);
logger.info("put resource : " + p.toUri());
FSDataOutputStream out = null;
- ResourceLock lock = null;
try {
- lock = lockManager.getLock(resPath);
- lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS);
out = fs.create(p, true);
out.writeLong(ts);
IOUtils.copy(content, out);
@@ -198,7 +176,6 @@ public class HDFSResourceStore extends ResourceStore {
throw new IOException("Put resource fail", e);
} finally {
IOUtils.closeQuietly(out);
- lockManager.releaseLock(lock);
}
}
@@ -222,18 +199,13 @@ public class HDFSResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- ResourceLock lock = null;
try {
- lock = lockManager.getLock(resPath);
- lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.SECONDS);
Path p = getRealHDFSPath(resPath);
if (fs.exists(p)) {
fs.delete(p, true);
}
} catch (Exception e) {
throw new IOException("Delete resource fail", e);
- } finally {
- lockManager.releaseLock(lock);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
index 4959718..582482f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java
@@ -31,6 +31,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.io.IOException;
import java.util.Arrays;
public class LockManager {
@@ -75,14 +76,14 @@ public class LockManager {
return new ResourceLock(lockPath, lock);
}
- public void releaseLock(ResourceLock lock) {
+ public void releaseLock(ResourceLock lock) throws IOException {
try {
if (lock != null)
lock.release();
} catch (Exception e) {
- logger.error("Fail to release lock");
- e.printStackTrace();
+ throw new IOException("Fail to release lock", e);
}
+
}
private static String getZKConnectString(KylinConfig kylinConfig) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/01abb00e/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
index 9d51871..ee5a415 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ResourceLock.java
@@ -33,8 +33,11 @@ public class ResourceLock {
this.lock = lock;
}
- public boolean acquire(long time, TimeUnit unit) throws Exception {
- return lock.acquire(time, unit);
+ public void acquire(long time, TimeUnit unit) throws Exception {
+ boolean success = lock.acquire(time, unit);
+ if(!success){
+ throw new IllegalStateException("Fail to get Zookeeper lock");
+ }
}
public void acquire() throws Exception{
[15/48] kylin git commit: Revert "KYLIN-2465 Web page has invalid
tips for streaming cube"
Posted by li...@apache.org.
Revert "KYLIN-2465 Web page has invalid tips for streaming cube"
This reverts commit cdfde7cb1ecb583af56dfb2f1255f819b1d93131.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20ab0d65
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20ab0d65
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20ab0d65
Branch: refs/heads/master-hbase0.98
Commit: 20ab0d65a2f7a47ff666baddbdf1d58b51356a63
Parents: 29dc1dd
Author: shaofengshi <sh...@apache.org>
Authored: Sun Mar 5 20:51:44 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 5 20:51:44 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/cubes/cubes.html | 1 +
.../app/partials/cubes/streaming_build_tip.html | 25 ++++++++++++++++++++
2 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/20ab0d65/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index 980ce9b..7fc3adb 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -137,6 +137,7 @@
</div>
<div ng-include="'partials/jobs/job_submit.html'"></div>
+<div ng-include="'partials/cubes/streaming_build_tip.html'"></div>
<div ng-include="'partials/jobs/job_refresh.html'"></div>
<div ng-include="'partials/jobs/job_merge.html'"></div>
<div ng-include="'partials/projects/project_create.html'"></div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/20ab0d65/webapp/app/partials/cubes/streaming_build_tip.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/streaming_build_tip.html b/webapp/app/partials/cubes/streaming_build_tip.html
new file mode 100644
index 0000000..80c5875
--- /dev/null
+++ b/webapp/app/partials/cubes/streaming_build_tip.html
@@ -0,0 +1,25 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+<script type="text/ng-template" id="streamingBuild.html">
+ <div class="callout callout-info">
+ <button type="button" class="close" data-dismiss="alert" ng-click="cancel()">�</button>
+ <h4><i class="fa fa-bullhorn"></i> Tips</h4>
+ Streaming cube build is not supported on UI, click <a href="{{streamingBuildUrl}}" style="color:blue;" target="_blank">here</a> to see how to schedule streaming cube.
+ </div>
+</script>
[34/48] kylin git commit: minor, refine kylin config
Posted by li...@apache.org.
minor, refine kylin config
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0301da01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0301da01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0301da01
Branch: refs/heads/master-hbase0.98
Commit: 0301da01b3669d92ad4bd07facfc8f79b7b0e0eb
Parents: 8edf234
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Mar 10 18:42:45 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 17:06:22 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigExt.java | 2 +-
.../security/PasswordPlaceholderConfigurer.java | 19 ++++++++++-
.../apache/kylin/rest/service/AdminService.java | 33 +++++++++++++++++++-
3 files changed, 51 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0301da01/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
index 6062d91..0ad24be 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
@@ -61,7 +61,7 @@ public class KylinConfigExt extends KylinConfig {
return super.getOptional(prop, dft);
}
- public Properties getAllProperties() {
+ protected Properties getAllProperties() {
Properties result = new Properties();
result.putAll(super.getAllProperties());
result.putAll(overrides);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0301da01/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 092d73a..5bb6d54 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.util.Properties;
@@ -30,6 +31,7 @@ import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigBase;
import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
@@ -51,7 +53,8 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
*/
public PasswordPlaceholderConfigurer() {
Resource[] resources = new Resource[1];
- Properties prop = KylinConfig.getKylinProperties();
+ //Properties prop = KylinConfig.getKylinProperties();
+ Properties prop = getAllKylinProperties();
StringWriter writer = new StringWriter();
prop.list(new PrintWriter(writer));
String propString = writer.getBuffer().toString();
@@ -61,6 +64,20 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
this.setLocations(resources);
}
+ public Properties getAllKylinProperties() {
+ // hack to get all config properties
+ Properties allProps = null;
+ try {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Method getAllMethod = KylinConfigBase.class.getDeclaredMethod("getAllProperties");
+ getAllMethod.setAccessible(true);
+ allProps = (Properties) getAllMethod.invoke(kylinConfig);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return allProps;
+ }
+
public static String encrypt(String strToEncrypt) {
try {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
http://git-wip-us.apache.org/repos/asf/kylin/blob/0301da01/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index a02804e..4cacb6b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
@@ -27,6 +28,7 @@ import java.util.TreeMap;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.util.OrderedProperties;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -94,7 +96,7 @@ public class AdminService extends BasicService {
logger.debug("Get Kylin Runtime Config");
try {
- return KylinConfig.getInstanceFromEnv().getConfigAsString();
+ return getAllConfigAsString();
} catch (IOException e) {
throw new InternalErrorException("Failed to get Kylin Runtime Config", e);
}
@@ -110,4 +112,33 @@ public class AdminService extends BasicService {
throw new InternalErrorException(e.getMessage(), e);
}
}
+
+ private String getAllConfigAsString() throws IOException {
+ Properties allProps;
+ try {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Method getAllMethod = KylinConfigBase.class.getDeclaredMethod("getAllProperties");
+ getAllMethod.setAccessible(true);
+ allProps = (Properties) getAllMethod.invoke(kylinConfig);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties();
+
+ final StringBuilder sb = new StringBuilder();
+
+ for (Map.Entry<Object, Object> entry : allProps.entrySet()) {
+ String key = entry.getKey().toString();
+ String value = entry.getValue().toString();
+ if (!orderedProperties.containsProperty(key)) {
+ orderedProperties.setProperty(key, value);
+ } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value))
+ orderedProperties.setProperty(key, value);
+ }
+ for (Map.Entry<String, String> entry : orderedProperties.entrySet()) {
+ sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
+ }
+ return sb.toString();
+ }
}
[25/48] kylin git commit: KYLIN-2491 Add job in error status into job
check filter when drop a cube.
Posted by li...@apache.org.
KYLIN-2491 Add job in error status into job check filter when drop a cube.
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f7885c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f7885c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f7885c1
Branch: refs/heads/master-hbase0.98
Commit: 4f7885c13727c9f6975c111a57138e75ca16de7a
Parents: f1154c5
Author: nichunen <ch...@kyligence.io>
Authored: Wed Mar 8 14:41:06 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 8 15:01:37 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 4 ++--
.../src/main/java/org/apache/kylin/rest/service/JobService.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f7885c1/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index c8c87cb..a28cedb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -248,9 +248,9 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
if (!cubingJobs.isEmpty()) {
- throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again.");
+ throw new JobException("The cube " + cube.getName() + " has running or failed job, please discard it and try again.");
}
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f7885c1/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index ed24a9d..893d866 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -485,7 +485,7 @@ public class JobService extends BasicService implements InitializingBean {
}
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
- return listAllCubingJobs(cubeName, projectName, statusList, -1L, -1L, allOutputs);
+ return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs);
}
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs) {
[27/48] kylin git commit: KYLIN-2477 bug fix
Posted by li...@apache.org.
KYLIN-2477 bug fix
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dfffde2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dfffde2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dfffde2
Branch: refs/heads/master-hbase0.98
Commit: 2dfffde267c91312fa9ff7644ecba46e3570e005
Parents: 01abb00
Author: xiefan46 <95...@qq.com>
Authored: Wed Mar 8 17:52:10 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 8 17:56:54 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/common/persistence/LocalFileResourceStoreTest.java | 2 +-
.../apache/kylin/common/persistence/ResourceStoreTest.java | 8 ++++----
.../apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java | 2 +-
.../apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 7 +++----
5 files changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfffde2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 917879d..02b2d4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -971,7 +971,7 @@ abstract public class KylinConfigBase implements Serializable {
public String getResourceStoreImpl() {
String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
int cut = metadataUrl.indexOf('@');
- String key = cut < 0 ? "" : metadataUrl.substring(0, cut);
+ String key = cut < 0 ? "" : metadataUrl.substring(cut + 1);
return getResourceStoreImpls().get(key);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfffde2/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
index db22d96..17b608d 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java
@@ -38,7 +38,7 @@ public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase {
@Test
public void testFileStore() throws Exception {
- ResourceStoreTest.testAStore("org.apache.kylin.common.persistence.FileResourceStore", "", KylinConfig.getInstanceFromEnv());
+ ResourceStoreTest.testAStore("", KylinConfig.getInstanceFromEnv());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfffde2/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 10714a5..76ab029 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -46,15 +46,15 @@ public class ResourceStoreTest {
private static final int TEST_RESOURCE_COUNT = 100;
- public static void testAStore(String storeName, String url, KylinConfig kylinConfig) throws Exception {
+ public static void testAStore(String url, KylinConfig kylinConfig) throws Exception {
String oldUrl = replaceMetadataUrl(kylinConfig, url);
- testAStore(getStoreByName(storeName, kylinConfig));
+ testAStore(getStoreByName(kylinConfig.getResourceStoreImpl(), kylinConfig));
replaceMetadataUrl(kylinConfig, oldUrl);
}
- public static void testPerformance(String storeName, String url, KylinConfig kylinConfig) throws Exception {
+ public static void testPerformance(String url, KylinConfig kylinConfig) throws Exception {
String oldUrl = replaceMetadataUrl(kylinConfig, url);
- testPerformance(getStoreByName(storeName, kylinConfig));
+ testPerformance(getStoreByName(kylinConfig.getResourceStoreImpl(), kylinConfig));
replaceMetadataUrl(kylinConfig, oldUrl);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfffde2/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
index 4e35118..74322bb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java
@@ -51,7 +51,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
@Test
public void testHBaseStore() throws Exception {
String storeName = "org.apache.kylin.storage.hbase.HBaseResourceStore";
- ResourceStoreTest.testAStore(storeName, ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig);
+ ResourceStoreTest.testAStore(ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfffde2/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 0a95f1c..0e21287 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -79,8 +79,7 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
}
private void doTestWithPath(String path) throws Exception {
- String storeName = "org.apache.kylin.storage.hdfs.HDFSResourceStore";
- ResourceStoreTest.testAStore(storeName, ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig);
+ ResourceStoreTest.testAStore(ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig);
assertTrue(fs.exists(new Path(path)));
}
@@ -90,10 +89,10 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
//test hdfs performance
String hdfsStoreName = "org.apache.kylin.storage.hdfs.HDFSResourceStore";
- ResourceStoreTest.testPerformance(hdfsStoreName, ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig);
+ ResourceStoreTest.testPerformance(ResourceStoreTest.mockUrl("hdfs", kylinConfig), kylinConfig);
//test hbase
String hbaseStoreName = "org.apache.kylin.storage.hbase.HBaseResourceStore.HBaseResourceStore";
- ResourceStoreTest.testPerformance(hbaseStoreName, ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig);
+ ResourceStoreTest.testPerformance(ResourceStoreTest.mockUrl("hbase", kylinConfig), kylinConfig);
}
}
[40/48] kylin git commit: KYLIN-2305 Unable to use long
searchBase/Pattern for LDAP
Posted by li...@apache.org.
KYLIN-2305 Unable to use long searchBase/Pattern for LDAP
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/181b6c5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/181b6c5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/181b6c5c
Branch: refs/heads/master-hbase0.98
Commit: 181b6c5c16fc0506a475993f3735a4b7e73a07c1
Parents: 0cb5032
Author: otomarukanta <kk...@yahoo-corp.jp>
Authored: Fri Mar 10 16:34:52 2017 +0900
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 22:12:55 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/security/PasswordPlaceholderConfigurer.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/181b6c5c/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 5bb6d54..095b74a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.security;
+import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -51,12 +52,12 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
/**
* The PasswordPlaceholderConfigurer will read Kylin properties as the Spring resource
*/
- public PasswordPlaceholderConfigurer() {
+ public PasswordPlaceholderConfigurer() throws IOException {
Resource[] resources = new Resource[1];
//Properties prop = KylinConfig.getKylinProperties();
Properties prop = getAllKylinProperties();
StringWriter writer = new StringWriter();
- prop.list(new PrintWriter(writer));
+ prop.store(new PrintWriter(writer), "kylin properties");
String propString = writer.getBuffer().toString();
IOUtils.closeQuietly(writer);
InputStream is = IOUtils.toInputStream(propString, Charset.defaultCharset());
[36/48] kylin git commit: cube desc json data modification
Posted by li...@apache.org.
cube desc json data modification
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/791a0100
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/791a0100
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/791a0100
Branch: refs/heads/master-hbase0.98
Commit: 791a010061ad216fd8bb4b5f31f1a1be5a43a141
Parents: 82b2236
Author: luguosheng <55...@qq.com>
Authored: Fri Mar 10 22:44:24 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 17:55:16 2017 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/791a0100/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 7e14d51..cfaadac 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -183,7 +183,7 @@
}
},
"returntype" : "topn(100, 4)",
- "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"}
}
}, {
"name" : "TEST_COUNT_DISTINCT_BITMAP",
http://git-wip-us.apache.org/repos/asf/kylin/blob/791a0100/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index b59d6df..221ffb6 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -183,7 +183,7 @@
}
},
"returntype" : "topn(100, 4)",
- "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"}
}
}, {
"name" : "TEST_COUNT_DISTINCT_BITMAP",
[32/48] kylin git commit: KYLIN-2495 query exception when integer
column encoded as date/time encoding
Posted by li...@apache.org.
KYLIN-2495 query exception when integer column encoded as date/time encoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5f3b228
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5f3b228
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5f3b228
Branch: refs/heads/master-hbase0.98
Commit: c5f3b2286987d12c31310e9c9670d469606e9d4c
Parents: 70dda40
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 10 15:57:10 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 17:17:02 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 16 ++---
.../apache/kylin/cube/model/RowKeyColDesc.java | 7 +--
.../kylin/dimension/AbstractDateDimEnc.java | 25 ++++----
.../org/apache/kylin/dimension/DateDimEnc.java | 64 ++++++++++++++++----
.../org/apache/kylin/dimension/TimeDimEnc.java | 13 ++--
.../kylin/measure/topn/TopNMeasureType.java | 12 +++-
.../apache/kylin/dimension/DateDimEncTest.java | 2 +-
7 files changed, 88 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 2412543..29858f1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -48,6 +48,10 @@ public class DateFormat {
}
return r;
}
+
+ public static String formatToCompactDateStr(long millis) {
+ return formatToDateStr(millis, COMPACT_DATE_PATTERN);
+ }
public static String formatToDateStr(long millis) {
return formatToDateStr(millis, DEFAULT_DATE_PATTERN);
@@ -88,18 +92,6 @@ public class DateFormat {
}
public static long stringToMillis(String str) {
- return stringToMillis(str, null);
- }
-
- public static long stringToMillis(String str, String dateFormat) {
- try {
- if (dateFormat != null) {
- return getDateFormat(dateFormat).parse(str).getTime();
- }
- } catch (ParseException e) {
- // given format does not work, proceed to below
- }
-
// try to be smart and guess the date format
if (isAllDigits(str)) {
if (str.length() == 8)
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 1df73ad..b6f0a27 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -86,10 +86,9 @@ public class RowKeyColDesc implements java.io.Serializable {
encoding = encodingName = TimeDimEnc.ENCODING_NAME;
}
}
- // if (DateDimEnc.ENCODING_NAME.equals(encodingName) && type.isDate() == false)
- // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply date encoding");
- // if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && type.isTimeFamily() == false)
- // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply time encoding");
+
+ encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type);
+
if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily()))
throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
index a54bcda..9c0d0da 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
@@ -18,10 +18,6 @@
package org.apache.kylin.dimension;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -29,20 +25,23 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
public class AbstractDateDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
- interface IMillisCodec extends Serializable {
- long millisToCode(long millis);
+ interface IValueCodec extends Serializable {
+ long valueToCode(String value);
- long codeToMillis(long code);
+ String codeToValue(long code);
}
// ============================================================================
private int fixedLen;
- private IMillisCodec codec;
+ private IValueCodec codec;
- protected AbstractDateDimEnc(int fixedLen, IMillisCodec codec) {
+ protected AbstractDateDimEnc(int fixedLen, IValueCodec codec) {
this.fixedLen = fixedLen;
this.codec = codec;
}
@@ -59,8 +58,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
return;
}
- long millis = DateFormat.stringToMillis(value);
- long code = codec.millisToCode(millis);
+ long code = codec.valueToCode(value);
BytesUtil.writeLong(code, output, outputOffset, fixedLen);
}
@@ -74,8 +72,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
if (code < 0)
throw new IllegalArgumentException();
- long millis = codec.codeToMillis(code);
- return String.valueOf(millis);
+ return codec.codeToValue(code);
}
@Override
@@ -137,7 +134,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
this.fixedLen = in.readInt();
- this.codec = (IMillisCodec) in.readObject();
+ this.codec = (IValueCodec) in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
index 3b390d9..fee6233 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
@@ -20,10 +20,13 @@ package org.apache.kylin.dimension;
import java.io.Serializable;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
+
/**
* This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility.
*/
-public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
+public class DateDimEnc extends AbstractDateDimEnc implements Serializable {
private static final long serialVersionUID = 1L;
public static final int ID_9999_12_31 = 3652426;
@@ -38,24 +41,47 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
@Override
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
- return new DateDimEnc();
+ return new DateDimEnc(args);
}
};
- public DateDimEnc() {
- super(3, new IMillisCodec() {
- private static final long serialVersionUID = 1L;
+ private static class DateDimValueCodec implements IValueCodec {
+
+ private static final long serialVersionUID = 1L;
+ private DataType datatype = null;
- @Override
- public long millisToCode(long millis) {
- return getNumOfDaysSince0000FromMillis(millis);
+ public DateDimValueCodec(String[] args) {
+ if (args != null && args.length == 1) {
+ datatype = DataType.getType(args[0]);
}
+ }
+
+ @Override
+ public long valueToCode(String value) {
+ //if data type is integer, DateFormat.stringToMillis recognizes format like "20001010"
+ long millis = DateFormat.stringToMillis(value);
+
+ return getNumOfDaysSince0000FromMillis(millis);
+ }
- @Override
- public long codeToMillis(long code) {
- return getMillisFromNumOfDaysSince0000(code);
+ @Override
+ public String codeToValue(long code) {
+ long millisFromNumOfDaysSince0000 = getMillisFromNumOfDaysSince0000(code);
+ if (datatype != null && datatype.isIntegerFamily()) {
+ return DateFormat.formatToCompactDateStr(millisFromNumOfDaysSince0000);
+ } else {
+ return String.valueOf(millisFromNumOfDaysSince0000);
}
- });
+ }
+ }
+
+ //keep this for ser/der
+ public DateDimEnc() {
+ super(3, new DateDimValueCodec(null));
+ }
+
+ public DateDimEnc(String[] args) {
+ super(3, new DateDimValueCodec(args));
}
public static long getNumOfDaysSince0000FromMillis(long millis) {
@@ -69,4 +95,18 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
return millis;
}
+ public static String[] replaceEncodingArgs(String encoding, String[] encodingArgs, String encodingName, DataType type) {
+ // https://issues.apache.org/jira/browse/KYLIN-2495
+ if (DateDimEnc.ENCODING_NAME.equals(encodingName)) {
+ if (type.isIntegerFamily()) {
+ if (encodingArgs.length != 0) {
+ throw new IllegalArgumentException("Date encoding should not specify arguments: " + encoding);
+ }
+ return new String[] { type.toString() };
+ }
+ }
+
+ return encodingArgs;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
index 4bef2b8..dcc99dd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
@@ -20,10 +20,12 @@ package org.apache.kylin.dimension;
import java.io.Serializable;
+import org.apache.kylin.common.util.DateFormat;
+
/**
* This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility.
*/
-public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
+public class TimeDimEnc extends AbstractDateDimEnc implements Serializable {
private static final long serialVersionUID = 1L;
public static final String ENCODING_NAME = "time";
@@ -41,17 +43,18 @@ public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
};
public TimeDimEnc() {
- super(4, new IMillisCodec() {
+ super(4, new IValueCodec() {
private static final long serialVersionUID = 1L;
@Override
- public long millisToCode(long millis) {
+ public long valueToCode(String value) {
+ long millis = DateFormat.stringToMillis(value);
return millis / 1000;
}
@Override
- public long codeToMillis(long code) {
- return code * 1000;
+ public String codeToValue(long code) {
+ return String.valueOf(code * 1000);
}
});
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 8c8b5a6..b7252a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -24,10 +24,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DateDimEnc;
import org.apache.kylin.dimension.DictionaryDimEnc;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -47,6 +47,8 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
@@ -155,7 +157,6 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return topNCounter;
}
-
@Override
public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
TopNCounter<ByteArray> topNCounter = value;
@@ -424,7 +425,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
}
Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
- dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion);
+ String encodingName = (String) encodingConf[0];
+ String[] encodingArgs = (String[]) encodingConf[1];
+
+ encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType());
+
+ dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
index 0183b5f..082f3ce 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
@@ -38,7 +38,7 @@ public class DateDimEncTest {
@Before
public void setup() {
- enc = new DateDimEnc();
+ enc = new DateDimEnc(null);
buf = new byte[enc.getLengthOfEncoding()];
}
[21/48] kylin git commit: KYLIN-2487 Do not convert large IN to
subquery join during query planning
Posted by li...@apache.org.
KYLIN-2487 Do not convert large IN to subquery join during query planning
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cde40f3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cde40f3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cde40f3f
Branch: refs/heads/master-hbase0.98
Commit: cde40f3f220e2cdde618c2500c05bdce9daf586b
Parents: 0e632fe
Author: lidongsjtu <li...@apache.org>
Authored: Mon Mar 6 21:55:22 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Mar 6 21:55:27 2017 +0800
----------------------------------------------------------------------
.../org/apache/calcite/sql2rel/SqlToRelConverter.java | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cde40f3f/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 02c45a0..2c308f5 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -193,7 +193,7 @@ import com.google.common.collect.Maps;
/*
* The code has synced with calcite. Hope one day, we could remove the hardcode override point.
* OVERRIDE POINT:
- * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
+ * - DEFAULT_IN_SUB_QUERY_THRESHOLD, was `20`, now `Integer.MAX_VALUE`
* - isTrimUnusedFields(), override to false
* - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
* - convertQuery(), call hackSelectStar() at the end
@@ -217,7 +217,8 @@ public class SqlToRelConverter {
/** Size of the smallest IN list that will be converted to a semijoin to a
* static table. */
- public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = 20;
+ /* OVERRIDE POINT */
+ public static final int DEFAULT_IN_SUB_QUERY_THRESHOLD = Integer.MAX_VALUE;
@Deprecated // to be removed before 2.0
public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
@@ -1017,7 +1018,7 @@ public class SqlToRelConverter {
scope.getValidator().deriveType(scope, e);
return e;
}
-
+
/**
* Converts a WHERE clause.
*
@@ -1532,9 +1533,7 @@ public class SqlToRelConverter {
*/
@Deprecated // to be removed before 2.0
protected int getInSubqueryThreshold() {
- //return config.getInSubQueryThreshold();
- /* OVERRIDE POINT */
- return Integer.MAX_VALUE;
+ return config.getInSubQueryThreshold();
}
/**
[39/48] kylin git commit: KYLIN-2384 backward compatible
Posted by li...@apache.org.
KYLIN-2384 backward compatible
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0cb5032e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0cb5032e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0cb5032e
Branch: refs/heads/master-hbase0.98
Commit: 0cb5032ee8da30a91f5ef8914138852daead558d
Parents: f38d871
Author: xiefan46 <95...@qq.com>
Authored: Sat Mar 11 16:46:54 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 22:09:55 2017 +0800
----------------------------------------------------------------------
.../kylin/dict/Number2BytesConverter.java | 4 ++
.../org/apache/kylin/dict/NumberDictionary.java | 12 ++++-
.../apache/kylin/dict/NumberDictionary2.java | 7 +++
.../org/apache/kylin/dict/TrieDictionary.java | 48 +++++++++-----------
4 files changed, 44 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
index 4e40527..814c95a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
@@ -53,6 +53,10 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
this.maxDigitsBeforeDecimalPoint = maxDigitsBeforeDecimalPoint;
}
+ public void setMaxDigitsBeforeDecimalPoint(int maxDigitsBeforeDecimalPoint) {
+ this.maxDigitsBeforeDecimalPoint = maxDigitsBeforeDecimalPoint;
+ }
+
@Override
public byte[] convertToBytes(String v) {
NumberBytesCodec codec = getCodec(this.maxDigitsBeforeDecimalPoint);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index b987eda..1377e8e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -19,6 +19,8 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.util.ClassUtil;
+
/**
* @author yangli9
*
@@ -32,6 +34,7 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
public NumberDictionary() { // default constructor for Writable interface
super();
+
}
public NumberDictionary(byte[] trieBytes) {
@@ -42,6 +45,13 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
protected boolean isNullObjectForm(T value) {
return value == null || value.equals("");
}
-
+
+ @Override
+ protected void setConverterByName(String converterName) throws Exception {
+ converterName = "org.apache.kylin.dict.Number2BytesConverter";
+ this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ ((Number2BytesConverter)this.bytesConvert).setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY);
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
index 3879d33..0efe936 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
@@ -18,6 +18,8 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.util.ClassUtil;
+
/**
* This class uses MAX_DIGITS_BEFORE_DECIMAL_POINT (=19) instead of legacy (=16).
*/
@@ -35,5 +37,10 @@ public class NumberDictionary2<T> extends NumberDictionary<T> {
super(trieBytes);
}
+ @Override
+ protected void setConverterByName(String converterName) throws Exception {
+ converterName = "org.apache.kylin.dict.Number2BytesConverter";
+ this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 9b84734..8849015 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -42,21 +42,21 @@ import com.google.common.base.Preconditions;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[] to
* int IDs.
- *
+ * <p>
* With Trie the memory footprint of the mapping is kinda minimized at the cost
* CPU, if compared to HashMap of ID Arrays. Performance test shows Trie is
* roughly 10 times slower, so there's a cache layer overlays on top of Trie and
* gracefully fall back to Trie using a weak reference.
- *
+ * <p>
* The implementation is thread-safe.
- *
+ *
* @author yangli9
*/
-@SuppressWarnings({ "rawtypes", "unchecked" })
+@SuppressWarnings({"rawtypes", "unchecked"})
public class TrieDictionary<T> extends CacheDictionary<T> {
private static final long serialVersionUID = 1L;
- public static final byte[] MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
+ public static final byte[] MAGIC = new byte[]{0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74}; // "TrieDict"
public static final int MAGIC_SIZE_I = MAGIC.length;
public static final int BIT_IS_LAST_CHILD = 0x80;
@@ -104,7 +104,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
String converterName = headIn.readUTF();
if (converterName.isEmpty() == false)
- this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ setConverterByName(converterName);
this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
@@ -119,6 +119,10 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
}
}
+ protected void setConverterByName(String converterName) throws Exception {
+ this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ }
+
@Override
public int getMinId() {
return baseId;
@@ -151,19 +155,14 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
/**
* returns a code point from [0, nValues), preserving order of value
- *
- * @param n
- * -- the offset of current node
- * @param inp
- * -- input value bytes to lookup
- * @param o
- * -- offset in the input value bytes matched so far
- * @param inpEnd
- * -- end of input
- * @param roundingFlag
- * -- =0: return -1 if not found
- * -- <0: return closest smaller if not found, return -1
- * -- >0: return closest bigger if not found, return nValues
+ *
+ * @param n -- the offset of current node
+ * @param inp -- input value bytes to lookup
+ * @param o -- offset in the input value bytes matched so far
+ * @param inpEnd -- end of input
+ * @param roundingFlag -- =0: return -1 if not found
+ * -- <0: return closest smaller if not found, return -1
+ * -- >0: return closest bigger if not found, return nValues
*/
private int lookupSeqNoFromValue(int n, byte[] inp, int o, int inpEnd, int roundingFlag) {
if (o == inpEnd) // special 'empty' value
@@ -257,13 +256,10 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
/**
* returns a code point from [0, nValues), preserving order of value, or -1
* if not found
- *
- * @param n
- * -- the offset of current node
- * @param seq
- * -- the code point under track
- * @param returnValue
- * -- where return value is written to
+ *
+ * @param n -- the offset of current node
+ * @param seq -- the code point under track
+ * @param returnValue -- where return value is written to
*/
private int lookupValueFromSeqNo(int n, int seq, byte[] returnValue, int offset) {
int o = offset;
[24/48] kylin git commit: KYLIN-2490 Couldn't get cube size on Azure
HDInsight
Posted by li...@apache.org.
KYLIN-2490 Couldn't get cube size on Azure HDInsight
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f1154c55
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f1154c55
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f1154c55
Branch: refs/heads/master-hbase0.98
Commit: f1154c55ebcc87280dc8d126e36587bc913dacc4
Parents: 67d30a0
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 8 11:23:29 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Mar 8 11:23:29 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/common/HadoopCmdOutput.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f1154c55/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
index 2a480e6..5da1947 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskCounter;
@@ -93,9 +94,16 @@ public class HadoopCmdOutput {
}
this.output.append(counters.toString()).append("\n");
logger.debug(counters.toString());
+
+ String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN";
+ String fsScheme = FileSystem.get(job.getConfiguration()).getScheme();
+ if (("wasb").equalsIgnoreCase(fsScheme)) {
+ // for Azure blob store
+ bytsWrittenCounterName = "WASB_BYTES_WRITTEN";
+ }
mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue());
- hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue());
+ hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue());
rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue());
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
[13/48] kylin git commit: KYLIN-2302 bug fix
Posted by li...@apache.org.
KYLIN-2302 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d2fa3ce
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d2fa3ce
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d2fa3ce
Branch: refs/heads/master-hbase0.98
Commit: 4d2fa3ce3eb65315cf9ec3769ddcb180004aa45f
Parents: 6c1f53e
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Mar 1 14:07:18 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 1 14:07:18 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/storage/StorageContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d2fa3ce/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 21a926b..f58fb34 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -64,7 +64,7 @@ public class StorageContext {
}
public int getLimit() {
- if (overlookOuterLimit || BackdoorToggles.getStatementMaxRows() == null) {
+ if (overlookOuterLimit || BackdoorToggles.getStatementMaxRows() == null || BackdoorToggles.getStatementMaxRows() == 0) {
return limit;
} else {
return Math.min(limit, BackdoorToggles.getStatementMaxRows());
[46/48] kylin git commit: KYLIN-2361 revert again because ordered
class loader is not guaranteed to work
Posted by li...@apache.org.
KYLIN-2361 revert again because ordered class loader is not guaranteed to work
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7976b5fc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7976b5fc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7976b5fc
Branch: refs/heads/master-hbase0.98
Commit: 7976b5fc714f5e73734b3037c05fc2601ea17662
Parents: f36b134
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Mar 12 17:18:27 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Mar 12 17:18:37 2017 +0800
----------------------------------------------------------------------
build/deploy/context.xml | 46 +--
build/deploy/server.xml | 63 ++--
build/script/download-tomcat.sh | 8 +-
pom.xml | 2 +-
.../java/org/apache/kylin/rest/DebugTomcat.java | 10 +-
.../kylin/ext/CustomizedWebappClassloader.java | 8 +-
.../kylin/ext/OrderedWebResourceRoot.java | 286 -------------------
.../kylin/ext/WebappOrderedClassLoader.java | 66 -----
8 files changed, 53 insertions(+), 436 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/build/deploy/context.xml
----------------------------------------------------------------------
diff --git a/build/deploy/context.xml b/build/deploy/context.xml
index da4777b..38c6ec8 100644
--- a/build/deploy/context.xml
+++ b/build/deploy/context.xml
@@ -1,32 +1,38 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version='1.0' encoding='utf-8'?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
<!-- The contents of this file will be loaded for each web application -->
<Context>
- <!-- Default set of monitored resources. If one of these changes, the -->
- <!-- web application will be reloaded. -->
+ <!-- Default set of monitored resources -->
<WatchedResource>WEB-INF/web.xml</WatchedResource>
- <WatchedResource>${catalina.base}/conf/web.xml</WatchedResource>
<!-- Uncomment this to disable session persistence across Tomcat restarts -->
<!--
<Manager pathname="" />
-->
+
+ <!-- Uncomment this to enable Comet connection tacking (provides events
+ on session expiration as well as webapp lifecycle) -->
+ <!--
+ <Valve className="org.apache.catalina.valves.CometConnectionManagerValve" />
+ -->
+
<Loader loaderClass="org.apache.kylin.ext.CustomizedWebappClassloader"/>
- <JarScanner scanClassPath="false" />
+
</Context>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/build/deploy/server.xml
----------------------------------------------------------------------
diff --git a/build/deploy/server.xml b/build/deploy/server.xml
index bea6146..42fab2d 100644
--- a/build/deploy/server.xml
+++ b/build/deploy/server.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version='1.0' encoding='utf-8'?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -20,12 +20,13 @@
Documentation at /docs/config/server.html
-->
<Server port="9005" shutdown="SHUTDOWN">
- <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
<!-- Security listener. Documentation at /docs/config/listeners.html
<Listener className="org.apache.catalina.security.SecurityListener" />
-->
<!--APR library loader. Documentation at /docs/apr.html -->
<Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
+ <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
+ <Listener className="org.apache.catalina.core.JasperListener" />
<!-- Prevent memory leaks due to use of particular java/javax APIs-->
<Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
<Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
@@ -61,61 +62,29 @@
<!-- A "Connector" represents an endpoint by which requests are received
and responses are returned. Documentation at :
- Java HTTP Connector: /docs/config/http.html
+ Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
Java AJP Connector: /docs/config/ajp.html
APR (HTTP/AJP) Connector: /docs/apr.html
- Define a non-SSL/TLS HTTP/1.1 Connector on port 8080
+ Define a non-SSL HTTP/1.1 Connector on port 8080
-->
<Connector port="7070" protocol="HTTP/1.1"
connectionTimeout="20000"
- redirectPort="9443"
+ redirectPort="7443"
compression="on"
compressionMinSize="2048"
noCompressionUserAgents="gozilla,traviata"
compressableMimeType="text/html,text/xml,text/javascript,application/javascript,application/json,text/css,text/plain"
/>
<!-- A "Connector" using the shared thread pool-->
- <!--
- <Connector executor="tomcatThreadPool"
- port="8080" protocol="HTTP/1.1"
- connectionTimeout="20000"
- redirectPort="8443" />
- -->
- <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443
- This connector uses the NIO implementation. The default
- SSLImplementation will depend on the presence of the APR/native
- library and the useOpenSSL attribute of the
- AprLifecycleListener.
- Either JSSE or OpenSSL style configuration may be used regardless of
- the SSLImplementation selected. JSSE style configuration is used below.
- -->
- <!--
- <Connector port="8443" protocol="org.apache.coyote.http11.Http11NioProtocol"
- maxThreads="150" SSLEnabled="true">
- <SSLHostConfig>
- <Certificate certificateKeystoreFile="conf/localhost-rsa.jks"
- type="RSA" />
- </SSLHostConfig>
- </Connector>
- -->
- <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443 with HTTP/2
- This connector uses the APR/native implementation which always uses
- OpenSSL for TLS.
- Either JSSE or OpenSSL style configuration may be used. OpenSSL style
- configuration is used below.
- -->
- <!--
- <Connector port="8443" protocol="org.apache.coyote.http11.Http11AprProtocol"
- maxThreads="150" SSLEnabled="true" >
- <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
- <SSLHostConfig>
- <Certificate certificateKeyFile="conf/localhost-rsa-key.pem"
- certificateFile="conf/localhost-rsa-cert.pem"
- certificateChainFile="conf/localhost-rsa-chain.pem"
- type="RSA" />
- </SSLHostConfig>
- </Connector>
- -->
+ <!-- Define a SSL HTTP/1.1 Connector on port 8443
+ This connector uses the BIO implementation that requires the JSSE
+ style configuration. When using the APR/native implementation, the
+ OpenSSL style configuration is required as described in the APR/native
+ documentation -->
+ <Connector port="7443" protocol="org.apache.coyote.http11.Http11Protocol"
+ maxThreads="150" SSLEnabled="true" scheme="https" secure="true"
+ keystoreFile="conf/.keystore" keystorePass="changeit"
+ clientAuth="false" sslProtocol="TLS" />
<!-- Define an AJP 1.3 Connector on port 8009 -->
<Connector port="9009" protocol="AJP/1.3" redirectPort="9443" />
@@ -163,7 +132,7 @@
Documentation at: /docs/config/valve.html
Note: The pattern used is equivalent to using pattern="common" -->
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log" suffix=".txt"
+ prefix="localhost_access_log." suffix=".txt"
pattern="%h %l %u %t "%r" %s %b" />
</Host>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/build/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index bdfe351..b3aa509 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,19 +27,19 @@ if [[ `uname -a` =~ "Darwin" ]]; then
alias md5cmd="md5 -q"
fi
-tomcat_pkg_version="8.5.9"
-tomcat_pkg_md5="b41270a64b7774c964e4bec813eea2ed"
+tomcat_pkg_version="7.0.69"
+tomcat_pkg_md5="10a071e5169a1a8b14ff35a0ad181052"
if [ ! -f "build/apache-tomcat-${tomcat_pkg_version}.tar.gz" ]
then
echo "no binary file found"
- wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "Download tomcat failed"
+ wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "Download tomcat failed"
else
if [ `md5cmd build/apache-tomcat-${tomcat_pkg_version}.tar.gz | awk '{print $1}'` != "${tomcat_pkg_md5}" ]
then
echo "md5 check failed"
rm build/apache-tomcat-${tomcat_pkg_version}.tar.gz
- wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-8/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "download tomcat failed"
+ wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-7/v${tomcat_pkg_version}/bin/apache-tomcat-${tomcat_pkg_version}.tar.gz || echo "download tomcat failed"
fi
fi
unalias md5cmd
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bb7fde9..30506e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,7 +110,7 @@
<cglib.version>3.2.4</cglib.version>
<supercsv.version>2.4.0</supercsv.version>
<cors.version>2.5</cors.version>
- <tomcat.version>8.5.9</tomcat.version>
+ <tomcat.version>7.0.69</tomcat.version>
<t-digest.version>3.1</t-digest.version>
<!-- REST Service -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 1b47f79..cd6768e 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -21,13 +21,11 @@ package org.apache.kylin.rest;
import org.apache.catalina.Context;
import org.apache.catalina.core.AprLifecycleListener;
import org.apache.catalina.core.StandardServer;
+import org.apache.catalina.deploy.ErrorPage;
import org.apache.catalina.startup.Tomcat;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
-import org.apache.tomcat.JarScanFilter;
-import org.apache.tomcat.JarScanType;
-import org.apache.tomcat.util.descriptor.web.ErrorPage;
import java.io.File;
import java.lang.reflect.Field;
@@ -129,12 +127,6 @@ public class DebugTomcat {
notFound.setLocation("/index.html");
webContext.addErrorPage(notFound);
webContext.addWelcomeFile("index.html");
- webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
- @Override
- public boolean check(JarScanType arg0, String arg1) {
- return false;
- }
- });
// tomcat start
tomcat.start();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index bbf4053..ec41842 100644
--- a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -18,12 +18,14 @@
package org.apache.kylin.ext;
+import org.apache.catalina.loader.ParallelWebappClassLoader;
+
/**
* simple extension to standard ParallelWebappClassLoader
* the only difference is that CustomizedWebappClassloader is able to delegate more packages
* to parent classloaders
*/
-public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
+public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
/**
* Set of package names which are not allowed to be loaded from a webapp
* class loader without delegating first.
@@ -43,7 +45,7 @@ public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
* @param name class name
* @return true if the class should be filtered
*/
- protected boolean filter(String name, boolean isClassName) {
+ protected boolean filter(String name) {
if (name == null)
return false;
@@ -60,6 +62,6 @@ public class CustomizedWebappClassloader extends WebappOrderedClassLoader {
return true;
}
- return super.filter(name, isClassName);
+ return super.filter(name);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
deleted file mode 100644
index 9784bd8..0000000
--- a/tomcat-ext/src/main/java/org/apache/kylin/ext/OrderedWebResourceRoot.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.ext;
-
-import java.io.InputStream;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.catalina.Context;
-import org.apache.catalina.LifecycleException;
-import org.apache.catalina.LifecycleListener;
-import org.apache.catalina.LifecycleState;
-import org.apache.catalina.TrackedWebResource;
-import org.apache.catalina.WebResource;
-import org.apache.catalina.WebResourceRoot;
-import org.apache.catalina.WebResourceSet;
-
-public class OrderedWebResourceRoot implements WebResourceRoot {
-
- private static final String WEB_INF_LIB_PATH = "/WEB-INF/lib";
-
- private static final Comparator<WebResource> WEB_RESOURCE_COMPARATOR = new Comparator<WebResource>() {
- @Override
- public int compare(WebResource o1, WebResource o2) {
- return o1.getName().compareTo(o2.getName());
- }
- };
-
- private WebResourceRoot delegate;
-
- public OrderedWebResourceRoot(WebResourceRoot delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public WebResource[] listResources(String path) {
- WebResource[] webResources = delegate.listResources(path);
-
- if (WEB_INF_LIB_PATH.equals(path)) {
- Arrays.sort(webResources, WEB_RESOURCE_COMPARATOR);
- }
-
- return webResources;
- }
-
- @Override
- public void addLifecycleListener(LifecycleListener listener) {
- delegate.addLifecycleListener(listener);
- }
-
- @Override
- public LifecycleListener[] findLifecycleListeners() {
- return delegate.findLifecycleListeners();
- }
-
- @Override
- public void removeLifecycleListener(LifecycleListener listener) {
- delegate.removeLifecycleListener(listener);
- }
-
- @Override
- public void init() throws LifecycleException {
- delegate.init();
- }
-
- @Override
- public void start() throws LifecycleException {
- delegate.start();
- }
-
- @Override
- public void stop() throws LifecycleException {
- delegate.stop();
- }
-
- @Override
- public void destroy() throws LifecycleException {
- delegate.destroy();
- }
-
- @Override
- public LifecycleState getState() {
- return delegate.getState();
- }
-
- @Override
- public String getStateName() {
- return delegate.getStateName();
- }
-
- @Override
- public WebResource getResource(String path) {
- return delegate.getResource(path);
- }
-
- @Override
- public WebResource[] getResources(String path) {
- return delegate.getResources(path);
- }
-
- @Override
- public WebResource getClassLoaderResource(String path) {
- return delegate.getClassLoaderResource(path);
- }
-
- @Override
- public WebResource[] getClassLoaderResources(String path) {
- return delegate.getClassLoaderResources(path);
- }
-
- @Override
- public String[] list(String path) {
- return delegate.list(path);
- }
-
- @Override
- public Set<String> listWebAppPaths(String path) {
- return delegate.listWebAppPaths(path);
- }
-
- @Override
- public boolean mkdir(String path) {
- return delegate.mkdir(path);
- }
-
- @Override
- public boolean write(String path, InputStream is, boolean overwrite) {
- return delegate.write(path, is, overwrite);
- }
-
- @Override
- public void createWebResourceSet(ResourceSetType type, String webAppMount, URL url, String internalPath) {
- delegate.createWebResourceSet(type, webAppMount, url, internalPath);
- }
-
- @Override
- public void createWebResourceSet(ResourceSetType type, String webAppMount, String base, String archivePath,
- String internalPath) {
- delegate.createWebResourceSet(type, webAppMount, base, archivePath, internalPath);
- }
-
- @Override
- public void addPreResources(WebResourceSet webResourceSet) {
- delegate.addPreResources(webResourceSet);
- }
-
- @Override
- public WebResourceSet[] getPreResources() {
- return delegate.getPreResources();
- }
-
- @Override
- public void addJarResources(WebResourceSet webResourceSet) {
- delegate.addJarResources(webResourceSet);
- }
-
- @Override
- public WebResourceSet[] getJarResources() {
- return delegate.getJarResources();
- }
-
- @Override
- public void addPostResources(WebResourceSet webResourceSet) {
- delegate.addPostResources(webResourceSet);
- }
-
- @Override
- public WebResourceSet[] getPostResources() {
- return delegate.getPostResources();
- }
-
- @Override
- public Context getContext() {
- return delegate.getContext();
- }
-
- @Override
- public void setContext(Context context) {
- delegate.setContext(context);
- }
-
- @Override
- public void setAllowLinking(boolean allowLinking) {
- delegate.setAllowLinking(allowLinking);
- }
-
- @Override
- public boolean getAllowLinking() {
- return delegate.getAllowLinking();
- }
-
- @Override
- public void setCachingAllowed(boolean cachingAllowed) {
- delegate.setCachingAllowed(cachingAllowed);
- }
-
- @Override
- public boolean isCachingAllowed() {
- return delegate.isCachingAllowed();
- }
-
- @Override
- public void setCacheTtl(long ttl) {
- delegate.setCacheTtl(ttl);
- }
-
- @Override
- public long getCacheTtl() {
- return delegate.getCacheTtl();
- }
-
- @Override
- public void setCacheMaxSize(long cacheMaxSize) {
- delegate.setCacheMaxSize(cacheMaxSize);
- }
-
- @Override
- public long getCacheMaxSize() {
- return delegate.getCacheMaxSize();
- }
-
- @Override
- public void setCacheObjectMaxSize(int cacheObjectMaxSize) {
- delegate.setCacheObjectMaxSize(cacheObjectMaxSize);
- }
-
- @Override
- public int getCacheObjectMaxSize() {
- return delegate.getCacheObjectMaxSize();
- }
-
- @Override
- public void setTrackLockedFiles(boolean trackLockedFiles) {
- delegate.setTrackLockedFiles(trackLockedFiles);
- }
-
- @Override
- public boolean getTrackLockedFiles() {
- return delegate.getTrackLockedFiles();
- }
-
- @Override
- public void backgroundProcess() {
- delegate.backgroundProcess();
- }
-
- @Override
- public void registerTrackedResource(TrackedWebResource trackedResource) {
- delegate.registerTrackedResource(trackedResource);
- }
-
- @Override
- public void deregisterTrackedResource(TrackedWebResource trackedResource) {
- delegate.deregisterTrackedResource(trackedResource);
- }
-
- @Override
- public List<URL> getBaseUrls() {
- return delegate.getBaseUrls();
- }
-
- @Override
- public void gc() {
- delegate.gc();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7976b5fc/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java
deleted file mode 100644
index 6a90e55..0000000
--- a/tomcat-ext/src/main/java/org/apache/kylin/ext/WebappOrderedClassLoader.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.ext;
-
-import org.apache.catalina.LifecycleException;
-import org.apache.catalina.WebResourceRoot;
-import org.apache.catalina.loader.ParallelWebappClassLoader;
-
-/**
- * Modified from the openwide-java/tomcat-classloader-ordered in https://github.com/openwide-java/tomcat-classloader-ordered
- *
- * This classloader is designed to return the jar of WEB-INF lib in alphabetical order as it was the case with Tomcat
- * 7.x.
- *
- * See the discussion in https://bz.apache.org/bugzilla/show_bug.cgi?id=57129 for more information.
- */
-public class WebappOrderedClassLoader extends ParallelWebappClassLoader {
-
- public WebappOrderedClassLoader() {
- }
-
- public WebappOrderedClassLoader(ClassLoader parent) {
- super(parent);
- }
-
- @Override
- public void setResources(WebResourceRoot resources) {
- super.setResources(new OrderedWebResourceRoot(resources));
- }
-
- @Override
- public WebappOrderedClassLoader copyWithoutTransformers() {
- WebappOrderedClassLoader result = new WebappOrderedClassLoader(getParent());
-
- super.copyStateWithoutTransformers(result);
-
- try {
- result.start();
- } catch (LifecycleException e) {
- throw new IllegalStateException(e);
- }
-
- return result;
- }
-
- @Override
- protected Object getClassLoadingLock(String className) {
- return this;
- }
-}
\ No newline at end of file
[14/48] kylin git commit: KYLIN-2481 fix ModelChooser
Posted by li...@apache.org.
KYLIN-2481 fix ModelChooser
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29dc1dd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29dc1dd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29dc1dd0
Branch: refs/heads/master-hbase0.98
Commit: 29dc1dd095ed70044f8f6a00e1ecfe8e0d1b0757
Parents: 4d2fa3c
Author: Li Yang <li...@apache.org>
Authored: Sat Mar 4 17:03:23 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Mar 4 17:09:39 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/query/routing/ModelChooser.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/29dc1dd0/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 507b371..f842345 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -163,7 +163,12 @@ public class ModelChooser {
TreeMap<DataModelDesc, Set<IRealization>> result = Maps.newTreeMap(new Comparator<DataModelDesc>() {
@Override
public int compare(DataModelDesc o1, DataModelDesc o2) {
- return costs.get(o1).compareTo(costs.get(o2));
+ RealizationCost c1 = costs.get(o1);
+ RealizationCost c2 = costs.get(o2);
+ int comp = c1.compareTo(c2);
+ if (comp == 0)
+ comp = o1.getName().compareTo(o2.getName());
+ return comp;
}
});
result.putAll(models);
[30/48] kylin git commit: KYLIN-2493 Auto enlarge buffer in
FactDistinctColumnsMapper
Posted by li...@apache.org.
KYLIN-2493 Auto enlarge buffer in FactDistinctColumnsMapper
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7eb9969e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7eb9969e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7eb9969e
Branch: refs/heads/master-hbase0.98
Commit: 7eb9969e1ce54cc919ad47f2589d85da44090d18
Parents: 182ed00
Author: xiefan46 <95...@qq.com>
Authored: Thu Mar 9 18:25:11 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 9 21:19:30 2017 +0800
----------------------------------------------------------------------
.../mr/steps/FactDistinctColumnsMapper.java | 24 +++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7eb9969e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index d9c1309..07e788c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -48,7 +48,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
public static enum RawDataCounter {
BYTES
- };
+ }
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
@@ -153,8 +153,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
tmpbuf.clear();
+ byte[] valueBytes = Bytes.toBytes(fieldValue);
+ int size = valueBytes.length + 1;
+ if (size >= tmpbuf.capacity()) {
+ tmpbuf = ByteBuffer.allocate(countNewSize(tmpbuf.capacity(), size));
+ }
tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
- tmpbuf.put(Bytes.toBytes(fieldValue));
+ tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
sortableKey.setText(outputKey);
//judge type
@@ -176,8 +181,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
String fieldValue = row[partitionColumnIndex];
if (fieldValue != null) {
tmpbuf.clear();
+ byte[] valueBytes = Bytes.toBytes(fieldValue);
+ int size = valueBytes.length + 1;
+ if (size >= tmpbuf.capacity()) {
+ tmpbuf = ByteBuffer.allocate(countNewSize(tmpbuf.capacity(), size));
+ }
tmpbuf.put(MARK_FOR_PARTITION_COL);
- tmpbuf.put(Bytes.toBytes(fieldValue));
+ tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
sortableKey.setText(outputKey);
sortableKey.setTypeId((byte) 0);
@@ -243,4 +253,12 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
}
}
+
+ private int countNewSize(int oldSize, int dataSize) {
+ int newSize = oldSize * 2;
+ while (newSize < dataSize) {
+ newSize = newSize * 2;
+ }
+ return newSize;
+ }
}
[20/48] kylin git commit: KYLIN-2483 solve bug for straming build
Posted by li...@apache.org.
KYLIN-2483 solve bug for straming build
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e632fe7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e632fe7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e632fe7
Branch: refs/heads/master-hbase0.98
Commit: 0e632fe75519966ef96313a51636736b1cc22bc5
Parents: 4d1f169
Author: luguosheng <55...@qq.com>
Authored: Mon Mar 6 17:51:19 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Mar 6 18:14:45 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubes.js | 53 +++++++++++++++++++++------------
1 file changed, 34 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e632fe7/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index cbdbb7d..6b59ef4 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -304,28 +304,43 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
if(table && table.source_type == 1){
cube.streaming = true;
}
-
// for streaming cube build tip
if(cube.streaming){
- $modal.open({
- templateUrl: 'streamingBuild.html',
- controller: streamingBuildCtrl,
- resolve: {
- cube: function () {
- return cube;
- },
- metaModel:function(){
- return $scope.metaModel;
- },
- buildType: function () {
- return 'BUILD';
- },
- scope:function(){
-
- return $scope;
- }
+ SweetAlert.swal({
+ title: '',
+ text: "Are you sure to start the build?",
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function(isConfirm) {
+ if(isConfirm){
+ loadingRequest.show();
+ CubeService.rebuildStreamingCube(
+ {
+ cubeId: cube.name
+ },
+ {
+ sourceOffsetStart:0,
+ sourceOffsetEnd:'9223372036854775807',
+ buildType:'BUILD'
+ }, function (job) {
+ loadingRequest.hide();
+ SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 'success');
+ },function(e){
+
+ loadingRequest.hide();
+ if(e.data&& e.data.exception){
+ var message =e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ }else{
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
}
- });
+ })
return;
}
[38/48] kylin git commit: KYLIN-2503 Spark cubing step should show
YARN app link
Posted by li...@apache.org.
KYLIN-2503 Spark cubing step should show YARN app link
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f38d871a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f38d871a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f38d871a
Branch: refs/heads/master-hbase0.98
Commit: f38d871a72e1e24331e092fd51664fcb8fd47e22
Parents: 5d012b9
Author: shaofengshi <sh...@apache.org>
Authored: Sat Mar 11 22:08:28 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 22:08:28 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/job/common/ShellExecutable.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f38d871a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index a68f242..5e0d0db 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -83,6 +83,10 @@ public class ShellExecutable extends AbstractExecutable {
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ // spark
+ private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
+ private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
+
@Override
public void log(String message) {
Matcher matcher = PATTERN_APP_ID.matcher(message);
@@ -136,6 +140,19 @@ public class ShellExecutable extends AbstractExecutable {
String hdfsWritten = matcher.group(2);
info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
}
+
+ // spark
+ matcher = PATTERN_SPARK_APP_ID.matcher(message);
+ if (matcher.find()) {
+ String app_id = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_ID, app_id);
+ }
+
+ matcher = PATTERN_SPARK_APP_URL.matcher(message);
+ if (matcher.find()) {
+ String trackingUrl = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+ }
}
Map<String, String> getInfo() {
[42/48] kylin git commit: fix BuildCubeWithStream test case may lose
record
Posted by li...@apache.org.
fix BuildCubeWithStream test case may lose record
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/530549cf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/530549cf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/530549cf
Branch: refs/heads/master-hbase0.98
Commit: 530549cf92c0f20d3247da563ca40ad4962c8ba4
Parents: 77fe581
Author: shaofengshi <sh...@apache.org>
Authored: Sun Mar 12 11:09:07 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 12 11:13:23 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/provision/BuildCubeWithStream.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/530549cf/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index f3b1ec9..2880bf4 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -93,7 +93,8 @@ public class BuildCubeWithStream {
private final String kafkaZkPath = "/" + UUID.randomUUID().toString();
protected static boolean fastBuildMode = false;
- private boolean generateData = true;
+ private volatile boolean generateData = true;
+ private volatile boolean generateDataDone = false;
private static final int BUILD_ROUND = 5;
@@ -182,11 +183,12 @@ public class BuildCubeWithStream {
try {
generateStreamData(dateStart, dateEnd, rand.nextInt(100));
dateStart = dateEnd;
- sleep(rand.nextInt(rand.nextInt(100 * 1000))); // wait random time
+ sleep(rand.nextInt(rand.nextInt(50)) * 1000); // wait random time
} catch (Exception e) {
e.printStackTrace();
}
}
+ generateDataDone = true;
}
}).start();
ExecutorService executorService = Executors.newCachedThreadPool();
@@ -197,7 +199,14 @@ public class BuildCubeWithStream {
// stop generating message to kafka
generateData = false;
}
- Thread.sleep(1 * 60 * 1000); // wait for new messages
+ int waittime = 0;
+ while (generateDataDone == false && waittime < 100) {
+ Thread.sleep(1000);
+ waittime++;
+ }
+ if (generateDataDone == false) {
+ throw new IllegalStateException("Timeout when wait all messages be sent to Kafka"); // ensure all messages have been flushed.
+ }
FutureTask futureTask = new FutureTask(new Callable<ExecutableState>() {
@Override
public ExecutableState call() {
[12/48] kylin git commit: minor, adopt json parse in CubeMetaExtractor
Posted by li...@apache.org.
minor, adopt json parse in CubeMetaExtractor
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c1f53ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c1f53ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c1f53ec
Branch: refs/heads/master-hbase0.98
Commit: 6c1f53ec9cdce3730b4c7262538dba698ddb5628
Parents: 7d57163
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Mar 1 12:08:06 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 1 13:53:36 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 38 +++++++++++++-------
1 file changed, 25 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c1f53ec/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 30f9736..35831f7 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import java.util.List;
import java.util.Set;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -60,6 +61,9 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -78,10 +82,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
@SuppressWarnings("static-access")
- private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageEngine").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageEngine");
+ private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageType");
@SuppressWarnings("static-access")
- private static final Option OPTION_ENGINE_TYPE = OptionBuilder.withArgName("typeEngine").hasArg().isRequired(false).withDescription("Specify the engine type to overwrite. Default is empty, keep origin.").create("typeEngine");
+ private static final Option OPTION_ENGINE_TYPE = OptionBuilder.withArgName("engineType").hasArg().isRequired(false).withDescription("Specify the engine type to overwrite. Default is empty, keep origin.").create("engineType");
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments");
@@ -143,8 +147,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false;
includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false;
onlyJobOutput = optionsHelper.hasOption(OPTION_INCLUDE_ONLY_JOB_OUTPUT) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_ONLY_JOB_OUTPUT)) : true;
- storageType = optionsHelper.hasOption(OPTION_STORAGE_TYPE) ? optionsHelper.getOptionValue(OPTION_STORAGE_TYPE): null;
- engineType = optionsHelper.hasOption(OPTION_ENGINE_TYPE) ? optionsHelper.getOptionValue(OPTION_ENGINE_TYPE): null;
+ storageType = optionsHelper.hasOption(OPTION_STORAGE_TYPE) ? optionsHelper.getOptionValue(OPTION_STORAGE_TYPE) : null;
+ engineType = optionsHelper.hasOption(OPTION_ENGINE_TYPE) ? optionsHelper.getOptionValue(OPTION_ENGINE_TYPE) : null;
kylinConfig = KylinConfig.getInstanceFromEnv();
metadataManager = MetadataManager.getInstance(kylinConfig);
@@ -253,17 +257,25 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
private void engineOverwriteInternal(File f) throws IOException {
- List<String> lines = FileUtils.readLines(f, "UTF-8");
- for (int i = 0, n = lines.size(); i < n; i++) {
- String l = lines.get(i);
- if (l.contains("\"engine_type\"")) {
- lines.set(i, " \"engine_type\" : " + engineType + ",");
+ try {
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode rootNode = objectMapper.readTree(f);
+ boolean replaced = false;
+ if (engineType != null && rootNode.get("engine_type") != null) {
+ ((ObjectNode) rootNode).put("engine_type", Integer.parseInt(engineType));
+ replaced = true;
+ }
+ if (storageType != null && rootNode.get("storage_type") != null) {
+ ((ObjectNode) rootNode).put("storage_type", Integer.parseInt(storageType));
+ replaced = true;
}
- if (l.contains("\"storage_type\"")) {
- lines.set(i, " \"storage_type\" : " + storageType + ",");
+ if (replaced) {
+ objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+ objectMapper.writeValue(f, rootNode);
}
+ } catch (JsonProcessingException ex) {
+ logger.info("cannot parse file {}", f);
}
- FileUtils.writeLines(f, "UTF-8", lines);
}
private IRealization getRealization(RealizationEntry realizationEntry) {
[37/48] kylin git commit: KYLIN-2499 HybridInstance cache wasn't
cleaned when underlying Cube be updated
Posted by li...@apache.org.
KYLIN-2499 HybridInstance cache wasn't cleaned when underlying Cube be updated
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d012b9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d012b9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d012b9f
Branch: refs/heads/master-hbase0.98
Commit: 5d012b9fa0ca6b3f98aa80f1ae9418be90d32bca
Parents: 791a010
Author: shaofengshi <sh...@apache.org>
Authored: Sat Mar 11 20:42:15 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 20:42:15 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/hybrid/HybridManager.java | 45 +++++++++++---------
1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d012b9f/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index 2d330c0..cf40416 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -88,14 +89,14 @@ public class HybridManager implements IRealizationProvider {
logger.info("Initializing HybridManager with config " + config);
this.config = config;
this.hybridMap = new CaseInsensitiveStringCache<HybridInstance>(config, "hybrid");
-
+
// touch lower level metadata before registering my listener
reloadAllHybridInstance();
- Broadcaster.getInstance(config).registerListener(new HybridSyncListener(), "hybrid");
+ Broadcaster.getInstance(config).registerListener(new HybridSyncListener(), "hybrid", "cube");
}
private class HybridSyncListener extends Broadcaster.Listener {
-
+
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
clearCache();
@@ -112,15 +113,22 @@ public class HybridManager implements IRealizationProvider {
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException {
- String hybridName = cacheKey;
-
- if (event == Event.DROP)
- hybridMap.removeLocal(hybridName);
- else
- reloadHybridInstance(hybridName);
-
- for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID, hybridName)) {
- broadcaster.notifyProjectSchemaUpdate(prj.getName());
+ if ("hybrid".equals(entity)) {
+ String hybridName = cacheKey;
+
+ if (event == Event.DROP)
+ hybridMap.removeLocal(hybridName);
+ else
+ reloadHybridInstance(hybridName);
+
+ for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID, hybridName)) {
+ broadcaster.notifyProjectSchemaUpdate(prj.getName());
+ }
+ } else if ("cube".equals(entity)) {
+ String cubeName = cacheKey;
+ for (HybridInstance hybrid : getHybridInstancesByChild(RealizationType.CUBE, cubeName)) {
+ reloadHybridInstance(hybrid.getName());
+ }
}
}
}
@@ -139,25 +147,24 @@ public class HybridManager implements IRealizationProvider {
logger.debug("Loaded " + paths.size() + " Hybrid(s)");
}
- public void reloadHybridInstanceByChild(RealizationType type, String realizationName) {
+ public List<HybridInstance> getHybridInstancesByChild(RealizationType type, String realizationName) {
+ List<HybridInstance> result = Lists.newArrayList();
for (HybridInstance hybridInstance : hybridMap.values()) {
- boolean includes = false;
for (RealizationEntry realizationEntry : hybridInstance.getRealizationEntries()) {
if (realizationEntry.getType() == type && realizationEntry.getRealization().equalsIgnoreCase(realizationName)) {
- includes = true;
- break;
+ result.add(hybridInstance);
}
}
- if (includes == true)
- reloadHybridInstance(hybridInstance.getName());
}
+
+ return result;
}
public void reloadHybridInstance(String name) {
reloadHybridInstanceAt(HybridInstance.concatResourcePath(name));
}
-
+
private synchronized HybridInstance reloadHybridInstanceAt(String path) {
ResourceStore store = getStore();
[05/48] kylin git commit: KYLIN-2331 bug fix
Posted by li...@apache.org.
KYLIN-2331 bug fix
temp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d6fc052e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d6fc052e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d6fc052e
Branch: refs/heads/master-hbase0.98
Commit: d6fc052e79bf669fd603ec3218b780c75208e292
Parents: bb27891
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Feb 25 13:50:20 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Feb 25 16:36:43 2017 +0800
----------------------------------------------------------------------
.../test/java/org/apache/kylin/provision/BuildCubeWithEngine.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d6fc052e/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 726d72f..df6c45e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -123,6 +123,7 @@ public class BuildCubeWithEngine {
System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
System.setProperty("SPARK_HOME", "/usr/local/spark"); // need manually create and put spark to this folder on Jenkins
+ System.setProperty("kylin.hadoop.conf.dir", HBaseMetadataTestCase.SANDBOX_TEST_DATA);
if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
}
[10/48] kylin git commit: KYLIN-2474 check lookup table after build
snapshot
Posted by li...@apache.org.
KYLIN-2474 check lookup table after build snapshot
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aed28ec4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aed28ec4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aed28ec4
Branch: refs/heads/master-hbase0.98
Commit: aed28ec4b315f760ce902794b8041c1239c467a3
Parents: a800122
Author: Li Yang <li...@apache.org>
Authored: Tue Feb 28 16:51:45 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Feb 28 21:30:48 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/cli/DictionaryGeneratorCLI.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aed28ec4/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 2e5a38e..e8c53f7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -32,6 +32,7 @@ import org.apache.kylin.dict.DictionaryProvider;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
@@ -77,16 +78,25 @@ public class DictionaryGeneratorCLI {
// snapshot
Set<String> toSnapshot = Sets.newHashSet();
+ Set<TableRef> toCheckLookup = Sets.newHashSet();
for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
TableRef table = dim.getTableRef();
- if (cubeSeg.getModel().isLookupTable(table))
+ if (cubeSeg.getModel().isLookupTable(table)) {
toSnapshot.add(table.getTableIdentity());
+ toCheckLookup.add(table);
+ }
}
for (String tableIdentity : toSnapshot) {
logger.info("Building snapshot of " + tableIdentity);
cubeMgr.buildSnapshotTable(cubeSeg, tableIdentity);
}
+
+ for (TableRef lookup : toCheckLookup) {
+ logger.info("Checking snapshot of " + lookup);
+ JoinDesc join = cubeSeg.getModel().getJoinsTree().getJoinByPKSide(lookup);
+ cubeMgr.getLookupTable(cubeSeg, join);
+ }
}
private static ReadableTable decideInputTable(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) {
[16/48] kylin git commit: KYLIN-2465 remove invalid tips for
streaming cube
Posted by li...@apache.org.
KYLIN-2465 remove invalid tips for streaming cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dd7507a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dd7507a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dd7507a5
Branch: refs/heads/master-hbase0.98
Commit: dd7507a5923a07baff99f72f88a86f74839f56eb
Parents: 20ab0d6
Author: shaofengshi <sh...@apache.org>
Authored: Sun Mar 5 20:53:26 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 5 20:53:26 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/cubes/streaming_build_tip.html | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dd7507a5/webapp/app/partials/cubes/streaming_build_tip.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/streaming_build_tip.html b/webapp/app/partials/cubes/streaming_build_tip.html
index 80c5875..7cdd2aa 100644
--- a/webapp/app/partials/cubes/streaming_build_tip.html
+++ b/webapp/app/partials/cubes/streaming_build_tip.html
@@ -19,7 +19,5 @@
<script type="text/ng-template" id="streamingBuild.html">
<div class="callout callout-info">
<button type="button" class="close" data-dismiss="alert" ng-click="cancel()">�</button>
- <h4><i class="fa fa-bullhorn"></i> Tips</h4>
- Streaming cube build is not supported on UI, click <a href="{{streamingBuildUrl}}" style="color:blue;" target="_blank">here</a> to see how to schedule streaming cube.
</div>
</script>
[09/48] kylin git commit: KYLIN-2457 Should copy the latest
dictionaries on dimension tables in a batch merge job
Posted by li...@apache.org.
KYLIN-2457 Should copy the latest dictionaries on dimension tables in a batch merge job
Signed-off-by: gaodayue <ga...@meituan.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8001226
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8001226
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8001226
Branch: refs/heads/master-hbase0.98
Commit: a8001226b2a07cd553e680b7e14de9bf8c9981f3
Parents: cda4df4
Author: zhengdong <zh...@outlook.com>
Authored: Tue Feb 21 19:56:48 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Tue Feb 28 20:03:55 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/steps/MergeDictionaryStep.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8001226/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
old mode 100644
new mode 100755
index 2281612..4ca132c
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -95,9 +95,9 @@ public class MergeDictionaryStep extends AbstractExecutable {
/**
* For the new segment, we need to create dictionaries for it, too. For
* those dictionaries on fact table, create it by merging underlying
- * dictionaries For those dictionaries on lookup table, just copy it from
- * any one of the merging segments, it's guaranteed to be consistent(checked
- * in CubeSegmentValidator)
+ * dictionaries. For those dictionaries on lookup table, just copy them from
+ * the latest one of the merging segments( https://issues.apache.org/jira/browse/KYLIN-2457),
+ * which is reasonable under the assumption that lookup tables would be either static or incremental.
*
* @param cube
* @param newSeg
@@ -135,9 +135,9 @@ public class MergeDictionaryStep extends AbstractExecutable {
}
mergeDictionaries(dictMgr, newSeg, dictInfos, col);
}
-
+ CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
for (TblColRef col : colsNeedCopyDict) {
- String path = mergingSegments.get(0).getDictResPath(col);
+ String path = lastSeg.getDictResPath(col);
newSeg.putDictResPath(col, path);
}
}
@@ -151,9 +151,9 @@ public class MergeDictionaryStep extends AbstractExecutable {
}
/**
- * make snapshots for the new segment by copying from one of the underlying
- * merging segments. it's guaranteed to be consistent(checked in
- * CubeSegmentValidator)
+ * make snapshots for the new segment by copying from the latest one of the underlying
+ * merging segments. It's guaranteed to be consistent under the assumption that lookup tables
+ * would be either static or incremental.
*
* @param cube
* @param newSeg
[04/48] kylin git commit: minor,
'hadoop fs' command add conf parameter
Posted by li...@apache.org.
minor, 'hadoop fs' command add conf parameter
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb278915
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb278915
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb278915
Branch: refs/heads/master-hbase0.98
Commit: bb2789156668f699d0c5f85bb578d5d0997fbde1
Parents: 693c6fa
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Feb 24 19:13:59 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Feb 24 19:49:02 2017 +0800
----------------------------------------------------------------------
build/bin/check-env.sh | 11 +++++++++--
build/bin/sample.sh | 13 ++++++++++---
2 files changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bb278915/build/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index e446d66..a0e91c4 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -18,6 +18,13 @@
#
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/find-hadoop-conf-dir.sh
+
+if [ -z "${kylin_hadoop_conf_dir}" ]; then
+ hadoop_conf_param=
+else
+ hadoop_conf_param="--config ${kylin_hadoop_conf_dir}"
+fi
if [ -z "$KYLIN_HOME" ]
then
@@ -42,13 +49,13 @@ then
fi
WORKING_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.env.hdfs-working-dir`
-hadoop fs -mkdir -p $WORKING_DIR
+hadoop ${hadoop_conf_param} fs -mkdir -p $WORKING_DIR
if [ $? != 0 ]
then
quit "Failed to create $WORKING_DIR. Please make sure the user has right to access $WORKING_DIR"
fi
-hadoop fs -mkdir -p $WORKING_DIR/spark-history
+hadoop ${hadoop_conf_param} fs -mkdir -p $WORKING_DIR/spark-history
if [ $? != 0 ]
then
quit "Failed to create $WORKING_DIR/spark-history. Please make sure the user has right to access $WORKING_DIR"
http://git-wip-us.apache.org/repos/asf/kylin/blob/bb278915/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 2c87956..4c65bcf 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -18,15 +18,22 @@
#
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/find-hadoop-conf-dir.sh
source ${dir}/check-env.sh "if-not-yet"
job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
cd ${KYLIN_HOME}/sample_cube/data
+if [ -z "${kylin_hadoop_conf_dir}" ]; then
+ hadoop_conf_param=
+else
+ hadoop_conf_param="--config ${kylin_hadoop_conf_dir}"
+fi
+
echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
-hadoop fs -mkdir -p /tmp/kylin/sample_cube/data
-hadoop fs -put * /tmp/kylin/sample_cube/data/
+hadoop ${hadoop_conf_param} fs -mkdir -p /tmp/kylin/sample_cube/data
+hadoop ${hadoop_conf_param} fs -put * /tmp/kylin/sample_cube/data/
hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.database-for-flat-table`
@@ -45,7 +52,7 @@ else
fi
echo "Sample hive tables are created successfully; Going to create sample cube..."
-hadoop fs -rm -r /tmp/kylin/sample_cube
+hadoop ${hadoop_conf_param} fs -rm -r /tmp/kylin/sample_cube
# set engine type and storage type to cube desc
default_engine_type=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.engine.default`
[17/48] kylin git commit: KYLIN-2483 SortedIteratorMergerWithLimit
could be slower when number of total merge rows is small
Posted by li...@apache.org.
KYLIN-2483 SortedIteratorMergerWithLimit could be slower when number of total merge rows is small
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3fc68437
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3fc68437
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3fc68437
Branch: refs/heads/master-hbase0.98
Commit: 3fc6843736f877712cd24ed8cafba4554b5e4d31
Parents: dd7507a
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Mar 5 22:44:14 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Mar 5 22:44:20 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 5 ++-
.../apache/kylin/storage/StorageContext.java | 45 +++++++++++++++-----
.../gtrecord/SequentialCubeTupleIterator.java | 21 +++------
.../gtrecord/StorageResponseGTScatter.java | 7 +--
.../org/apache/kylin/query/KylinTestBase.java | 2 +-
5 files changed, 50 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fc68437/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index df3e914..3a87d3d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -773,7 +773,6 @@ abstract public class KylinConfigBase implements Serializable {
// ENGINE.SPARK
// ============================================================================
-
public String getHadoopConfDir() {
return getOptional("kylin.env.hadoop-conf-dir", "");
}
@@ -817,6 +816,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
}
+ public int getMergeSortPartitionResultsMinLimit() {
+ return Integer.parseInt(getOptional("kylin.query.merge-sort-partition-results.min-limit", "100"));
+ }
+
public long getQueryMaxScanBytes() {
long value = Long.parseLong(getOptional("kylin.query.max-scan-bytes", "0"));
return value > 0 ? value : Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fc68437/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index f58fb34..f4211ff 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -20,9 +20,11 @@ package org.apache.kylin.storage;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.storage.gtrecord.GTCubeStorageQueryBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +47,6 @@ public class StorageContext {
private boolean exactAggregation = false;
private boolean needStorageAggregation = false;
- private boolean limitEnabled = false;
private boolean enableCoprocessor = false;
private IStorageQuery storageQuery;
@@ -63,7 +64,9 @@ public class StorageContext {
this.connUrl = connUrl;
}
- public int getLimit() {
+ //the limit here correspond to the limit concept in SQL
+ //also take into consideration Statement.setMaxRows in JDBC
+ private int getLimit() {
if (overlookOuterLimit || BackdoorToggles.getStatementMaxRows() == null || BackdoorToggles.getStatementMaxRows() == 0) {
return limit;
} else {
@@ -71,10 +74,6 @@ public class StorageContext {
}
}
- public void setOverlookOuterLimit() {
- this.overlookOuterLimit = true;
- }
-
public void setLimit(int l) {
if (limit != Integer.MAX_VALUE) {
logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit);
@@ -83,6 +82,12 @@ public class StorageContext {
}
}
+ //outer limit is sth like Statement.setMaxRows in JDBC
+ public void setOverlookOuterLimit() {
+ this.overlookOuterLimit = true;
+ }
+
+ //the offset here correspond to the offset concept in SQL
public int getOffset() {
return offset;
}
@@ -91,8 +96,18 @@ public class StorageContext {
this.offset = offset;
}
- public boolean isLimitEnabled() {
- return this.limitEnabled;
+ /**
+ * in contrast to the limit in SQL concept, "limit push down" means
+ * whether the limit is effective in storage level. Some queries are not possible
+ * to leverage limit clause, checkout
+ * {@link GTCubeStorageQueryBase#enableStorageLimitIfPossible(org.apache.kylin.cube.cuboid.Cuboid, java.util.Collection, java.util.Set, java.util.Collection, org.apache.kylin.metadata.filter.TupleFilter, java.util.Set, java.util.Collection, org.apache.kylin.storage.StorageContext)}
+ */
+ public boolean isLimitPushDownEnabled() {
+ return isValidPushDownLimit(finalPushDownLimit);
+ }
+
+ public static boolean isValidPushDownLimit(int finalPushDownLimit) {
+ return finalPushDownLimit < Integer.MAX_VALUE && finalPushDownLimit > 0;
}
public int getFinalPushDownLimit() {
@@ -101,7 +116,7 @@ public class StorageContext {
public void setFinalPushDownLimit(IRealization realization) {
- if (this.getLimit() == Integer.MAX_VALUE) {
+ if (!isValidPushDownLimit(this.getLimit())) {
return;
}
@@ -110,12 +125,20 @@ public class StorageContext {
if (!realization.supportsLimitPushDown()) {
logger.warn("Not enabling limit push down because cube storage type not supported");
} else {
- this.limitEnabled = true;
this.finalPushDownLimit = tempPushDownLimit;
- logger.info("Enable limit: " + tempPushDownLimit);
+ logger.info("Enable limit (storage push down limit) :" + tempPushDownLimit);
}
}
+ public boolean mergeSortPartitionResults() {
+ return mergeSortPartitionResults(finalPushDownLimit);
+ }
+
+ public static boolean mergeSortPartitionResults(int finalPushDownLimit) {
+ return isValidPushDownLimit(finalPushDownLimit) && //
+ (finalPushDownLimit > KylinConfig.getInstanceFromEnv().getMergeSortPartitionResultsMinLimit());
+ }
+
public long getDeadline() {
return this.deadline;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fc68437/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index 14b6394..9d5d816 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -24,8 +24,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import javax.annotation.Nullable;
-
import org.apache.kylin.common.exceptions.KylinTimeoutException;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -37,7 +35,6 @@ import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@@ -64,19 +61,15 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
}
- if (!context.isLimitEnabled()) {
- //normal case
- tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
- } else {
+ if (context.mergeSortPartitionResults()) {
//query with limit
- Iterator<Iterator<ITuple>> transformed = Iterators.transform(segmentCubeTupleIterators.iterator(), new Function<SegmentCubeTupleIterator, Iterator<ITuple>>() {
- @Nullable
- @Override
- public Iterator<ITuple> apply(@Nullable SegmentCubeTupleIterator input) {
- return input;
- }
- });
+ logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
+ Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator();
+ } else {
+ //normal case
+ logger.info("Using Iterators.concat to merge segment results");
+ tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fc68437/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 3904b5c..1a80bbf 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,11 +70,11 @@ public class StorageResponseGTScatter implements IGTScanner {
@Override
public Iterator<GTRecord> iterator() {
Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
- if (storagePushDownLimit != Integer.MAX_VALUE) {
- logger.info("Using SortedIteratorMergerWithLimit to merge partitions");
+ if (StorageContext.mergeSortPartitionResults(storagePushDownLimit)) {
+ logger.info("Using SortedIteratorMergerWithLimit to merge partition results");
return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
} else {
- logger.info("Using Iterators.concat to merge partitions");
+ logger.info("Using Iterators.concat to merge partition results");
return Iterators.concat(shardSubsets);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3fc68437/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index d0524c6..2174094 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -682,7 +682,7 @@ public class KylinTestBase {
protected boolean checkFinalPushDownLimit() {
OLAPContext context = getFirstOLAPContext();
- return context.storageContext.isLimitEnabled();
+ return context.storageContext.isLimitPushDownEnabled();
}
[19/48] kylin git commit: KYLIN-2484 add validation for streaming +
spark combination
Posted by li...@apache.org.
KYLIN-2484 add validation for streaming + spark combination
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d1f169c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d1f169c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d1f169c
Branch: refs/heads/master-hbase0.98
Commit: 4d1f169cd883319491c72a68b6ca8428fb217cfa
Parents: ac35cd3
Author: shaofengshi <sh...@apache.org>
Authored: Mon Mar 6 11:48:53 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Mar 6 11:49:06 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/model/validation/rule/StreamingCubeRule.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d1f169c/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
index fb0db8c..4438706 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/StreamingCubeRule.java
@@ -24,6 +24,7 @@ import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.TblColRef;
@@ -53,6 +54,11 @@ public class StreamingCubeRule implements IValidatorRule<CubeDesc> {
return;
}
+ if (cube.getEngineType() == IEngineAware.ID_SPARK) {
+ context.addResult(ResultLevel.ERROR, "Spark engine doesn't support streaming source, select MapReduce engine instead.");
+ return;
+ }
+
if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
context.addResult(ResultLevel.ERROR, "Must define a partition column.");
return;
[23/48] kylin git commit: minor, update some api visibility
Posted by li...@apache.org.
minor, update some api visibility
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67d30a02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67d30a02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67d30a02
Branch: refs/heads/master-hbase0.98
Commit: 67d30a023e75a76dd5d5e146d42f50f6fa749a2f
Parents: a21ffc8
Author: lidongsjtu <li...@apache.org>
Authored: Tue Mar 7 20:33:37 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Mar 7 20:36:26 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/67d30a02/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index a72460c..ecf1ad3 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -62,8 +62,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
private static final Logger logger = LoggerFactory.getLogger(GTCubeStorageQueryBase.class);
- private final CubeInstance cubeInstance;
- private final CubeDesc cubeDesc;
+ protected final CubeInstance cubeInstance;
+ protected final CubeDesc cubeDesc;
public GTCubeStorageQueryBase(CubeInstance cube) {
this.cubeInstance = cube;
@@ -148,7 +148,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
}
- private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
+ protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
for (FunctionDesc func : sqlDigest.aggregations) {
if (!func.isDimensionAsMetric()) {
// use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision
@@ -174,7 +174,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return aggrFunc;
}
- private Set<TblColRef> expandDerived(Collection<TblColRef> cols, Set<TblColRef> derivedPostAggregation) {
+ protected Set<TblColRef> expandDerived(Collection<TblColRef> cols, Set<TblColRef> derivedPostAggregation) {
Set<TblColRef> expanded = Sets.newHashSet();
for (TblColRef col : cols) {
if (cubeDesc.hasHostColumn(col)) {
@@ -253,7 +253,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
@SuppressWarnings("unchecked")
- private TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
+ protected TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
if (filter == null)
return filter;
@@ -366,7 +366,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
- private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
+ protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
Map<String, List<MeasureDesc>> map = Maps.newHashMap();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
MeasureType<?> measureType = measure.getFunction().getMeasureType();
[45/48] kylin git commit: stabilize BuildCubeWithStream
Posted by li...@apache.org.
stabilize BuildCubeWithStream
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f36b1341
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f36b1341
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f36b1341
Branch: refs/heads/master-hbase0.98
Commit: f36b1341b7905e68b99afc77c8009ba156c0bca4
Parents: 47b7c80
Author: shaofengshi <sh...@apache.org>
Authored: Sun Mar 12 15:15:13 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 12 15:15:13 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/provision/BuildCubeWithStream.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f36b1341/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index c5341d6..a527839 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -183,7 +183,7 @@ public class BuildCubeWithStream {
try {
generateStreamData(dateStart, dateEnd, rand.nextInt(100));
dateStart = dateEnd;
- sleep(rand.nextInt(rand.nextInt(50)) * 1000); // wait random time
+ sleep(rand.nextInt(rand.nextInt(30)) * 1000); // wait random time
} catch (Exception e) {
e.printStackTrace();
}
@@ -206,6 +206,8 @@ public class BuildCubeWithStream {
if (generateDataDone == false) {
throw new IllegalStateException("Timeout when wait all messages be sent to Kafka"); // ensure all messages have been flushed.
}
+ } else {
+ Thread.sleep(30 * 1000); // wait for new messages
}
FutureTask futureTask = new FutureTask(new Callable<ExecutableState>() {
@@ -290,8 +292,8 @@ public class BuildCubeWithStream {
protected void deployEnv() throws IOException {
DeployUtil.overrideJobJarLocations();
- // DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
+ // DeployUtil.initCliWorkDir();
+ // DeployUtil.deployMetadata();
}
public static void beforeClass() throws Exception {
[07/48] kylin git commit: minor,
CubeMetaExtractor can change storage and engine type
Posted by li...@apache.org.
minor, CubeMetaExtractor can change storage and engine type
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e875e870
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e875e870
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e875e870
Branch: refs/heads/master-hbase0.98
Commit: e875e87042c7b78dbf452c4ec158743c1be6769e
Parents: 29b3f6d
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Feb 28 11:00:55 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Feb 28 11:04:52 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 41 ++++++++++++++++++++
1 file changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e875e870/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 0067e24..30f9736 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -19,12 +19,14 @@
package org.apache.kylin.tool;
import java.io.File;
+import java.io.IOException;
import java.util.List;
import java.util.Set;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -76,6 +78,12 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
@SuppressWarnings("static-access")
+ private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageEngine").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageEngine");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_ENGINE_TYPE = OptionBuilder.withArgName("typeEngine").hasArg().isRequired(false).withDescription("Specify the engine type to overwrite. Default is empty, keep origin.").create("typeEngine");
+
+ @SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments");
@SuppressWarnings("static-access")
@@ -102,6 +110,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private boolean includeJobs;
private boolean includeSegmentDetails;
private boolean onlyJobOutput;
+ private String storageType = null;
+ private String engineType = null;
private Set<String> requiredResources = Sets.newLinkedHashSet();
private Set<String> optionalResources = Sets.newLinkedHashSet();
@@ -123,6 +133,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
options.addOption(OPTION_INCLUDE_JOB);
options.addOption(OPTION_INCLUDE_SEGMENT_DETAILS);
options.addOption(OPTION_INCLUDE_ONLY_JOB_OUTPUT);
+ options.addOption(OPTION_STORAGE_TYPE);
+ options.addOption(OPTION_ENGINE_TYPE);
}
@Override
@@ -131,6 +143,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false;
includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false;
onlyJobOutput = optionsHelper.hasOption(OPTION_INCLUDE_ONLY_JOB_OUTPUT) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_ONLY_JOB_OUTPUT)) : true;
+ storageType = optionsHelper.hasOption(OPTION_STORAGE_TYPE) ? optionsHelper.getOptionValue(OPTION_STORAGE_TYPE): null;
+ engineType = optionsHelper.hasOption(OPTION_ENGINE_TYPE) ? optionsHelper.getOptionValue(OPTION_ENGINE_TYPE): null;
kylinConfig = KylinConfig.getInstanceFromEnv();
metadataManager = MetadataManager.getInstance(kylinConfig);
@@ -184,6 +198,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
executeExtraction(exportDir.getAbsolutePath());
+ engineOverwrite(new File(exportDir.getAbsolutePath()));
}
private void executeExtraction(String dest) {
@@ -225,6 +240,32 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
}
+ private void engineOverwrite(File dest) throws IOException {
+ if (engineType != null || storageType != null) {
+ for (File f : dest.listFiles()) {
+ if (f.isDirectory()) {
+ engineOverwrite(f);
+ } else {
+ engineOverwriteInternal(f);
+ }
+ }
+ }
+ }
+
+ private void engineOverwriteInternal(File f) throws IOException {
+ List<String> lines = FileUtils.readLines(f, "UTF-8");
+ for (int i = 0, n = lines.size(); i < n; i++) {
+ String l = lines.get(i);
+ if (l.contains("\"engine_type\"")) {
+ lines.set(i, " \"engine_type\" : " + engineType + ",");
+ }
+ if (l.contains("\"storage_type\"")) {
+ lines.set(i, " \"storage_type\" : " + storageType + ",");
+ }
+ }
+ FileUtils.writeLines(f, "UTF-8", lines);
+ }
+
private IRealization getRealization(RealizationEntry realizationEntry) {
return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
}
[48/48] kylin git commit: KYLIN-2307 Create a branch for master with
HBase 0.98 API
Posted by li...@apache.org.
KYLIN-2307 Create a branch for master with HBase 0.98 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1877487f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1877487f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1877487f
Branch: refs/heads/master-hbase0.98
Commit: 1877487fa1aacb5c1412b81f78823fd26233c1fb
Parents: 7976b5f
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Mar 12 19:46:54 2017 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25 ++++
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25 ----
.../sandbox/capacity-scheduler.xml | 17 ++-
examples/test_case_data/sandbox/core-site.xml | 28 +---
examples/test_case_data/sandbox/hbase-site.xml | 119 +++++------------
examples/test_case_data/sandbox/hdfs-site.xml | 84 +++++-------
examples/test_case_data/sandbox/hive-site.xml | 89 +++++--------
examples/test_case_data/sandbox/mapred-site.xml | 57 +++------
examples/test_case_data/sandbox/yarn-site.xml | 127 +++----------------
.../kylin/provision/BuildCubeWithEngine.java | 17 +--
pom.xml | 117 +----------------
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 +++++++++++---
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 ++--
.../apache/kylin/rest/service/CubeService.java | 35 +++--
.../apache/kylin/rest/service/QueryService.java | 24 ++--
.../apache/kylin/rest/service/UserService.java | 17 +--
.../kylin/storage/hbase/HBaseConnection.java | 44 +++----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++--
.../storage/hbase/cube/SimpleHBaseStore.java | 20 +--
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 +--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++--
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--
.../storage/hbase/util/CleanHtableCLI.java | 12 +-
.../storage/hbase/util/CubeMigrationCLI.java | 37 +++---
.../hbase/util/CubeMigrationCheckCLI.java | 17 +--
.../hbase/util/DeployCoprocessorCLI.java | 27 ++--
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 ++---
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++-
.../hbase/util/HBaseRegionSizeCalculator.java | 35 +++--
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 ++---
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-
.../storage/hbase/util/StorageCleanupJob.java | 20 ++-
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 +--
tool/pom.xml | 10 --
.../org/apache/kylin/tool/CubeMigrationCLI.java | 16 ++-
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
48 files changed, 595 insertions(+), 870 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 0000000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/dev-support/test_all_against_hdp_2_4_0_0_169.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
deleted file mode 100755
index 2a3d24b..0000000
--- a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index e042aa5..7cb985c 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -47,6 +47,16 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.acl_administer_queue</name>
<value>*</value>
</property>
@@ -57,6 +67,11 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
+ <value></value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
<value>*</value>
</property>
@@ -96,4 +111,4 @@
<value>default</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index a4ad5c6..0c5f62b 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -19,6 +19,7 @@
<property>
<name>fs.defaultFS</name>
<value>hdfs://sandbox.hortonworks.com:8020</value>
+ <final>true</final>
</property>
<property>
@@ -38,7 +39,7 @@
<property>
<name>hadoop.proxyuser.falcon.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
@@ -48,7 +49,7 @@
<property>
<name>hadoop.proxyuser.hbase.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
@@ -67,23 +68,13 @@
</property>
<property>
- <name>hadoop.proxyuser.hdfs.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hdfs.hosts</name>
- <value>*</value>
- </property>
-
- <property>
<name>hadoop.proxyuser.hive.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <value>*</value>
</property>
<property>
@@ -132,15 +123,8 @@
</property>
<property>
- <name>hadoop.security.key.provider.path</name>
- <value></value>
- </property>
-
- <property>
<name>io.compression.codecs</name>
- <value>
- org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec
- </value>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 568de2e..46d5345 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -22,33 +22,8 @@
</property>
<property>
- <name>hbase.bucketcache.ioengine</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bucketcache.percentage.in.combinedcache</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bucketcache.size</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bulkload.staging.dir</name>
- <value>/apps/hbase/staging</value>
- </property>
-
- <property>
<name>hbase.client.keyvalue.maxsize</name>
- <value>1048576</value>
- </property>
-
- <property>
- <name>hbase.client.retries.number</name>
- <value>35</value>
+ <value>10485760</value>
</property>
<property>
@@ -63,19 +38,12 @@
<property>
<name>hbase.coprocessor.master.classes</name>
- <value>org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor</value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>
- org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor
- </value>
- </property>
-
- <property>
- <name>hbase.coprocessor.regionserver.classes</name>
- <value></value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
@@ -119,11 +87,6 @@
</property>
<property>
- <name>hbase.hstore.compaction.max</name>
- <value>10</value>
- </property>
-
- <property>
<name>hbase.hstore.compactionThreshold</name>
<value>3</value>
</property>
@@ -140,42 +103,32 @@
<property>
<name>hbase.master.info.port</name>
- <value>16010</value>
+ <value>60010</value>
</property>
<property>
<name>hbase.master.port</name>
- <value>16000</value>
+ <value>60000</value>
</property>
<property>
- <name>hbase.region.server.rpc.scheduler.factory.class</name>
- <value></value>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
</property>
<property>
- <name>hbase.regionserver.global.memstore.size</name>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
<value>0.4</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
- <value>30</value>
+ <value>60</value>
</property>
<property>
<name>hbase.regionserver.info.port</name>
- <value>16030</value>
- </property>
-
- <property>
- <name>hbase.regionserver.port</name>
- <value>16020</value>
- </property>
-
- <property>
- <name>hbase.regionserver.wal.codec</name>
- <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value>
+ <value>60030</value>
</property>
<property>
@@ -184,26 +137,11 @@
</property>
<property>
- <name>hbase.rpc.controllerfactory.class</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.rpc.engine</name>
- <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value>
- </property>
-
- <property>
<name>hbase.rpc.protection</name>
<value>PRIVACY</value>
</property>
<property>
- <name>hbase.rpc.timeout</name>
- <value>90000</value>
- </property>
-
- <property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
@@ -220,7 +158,7 @@
<property>
<name>hbase.tmp.dir</name>
- <value>/tmp/hbase-${user.name}</value>
+ <value>/hadoop/hbase</value>
</property>
<property>
@@ -240,27 +178,34 @@
<property>
<name>hfile.block.cache.size</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>phoenix.functions.allowUserDefinedFunctions</name>
- <value></value>
- </property>
-
- <property>
- <name>phoenix.query.timeoutMs</name>
- <value>60000</value>
+ <value>0.40</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
- <value>60000</value>
+ <value>30000</value>
</property>
<property>
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
-
-</configuration>
\ No newline at end of file
+ <property>
+ <name>hbase.client.pause</name>
+ <value>100</value>
+ <description>General client pause value. Used mostly as value to wait
+ before running a retry of a failed get, region lookup, etc.
+ See hbase.client.retries.number for description of how we backoff from
+ this initial pause amount and how this pause works w/ retries.</description>
+ </property>
+ <property>
+ <name>hbase.client.retries.number</name>
+ <value>5</value>
+ <description>Maximum retries. Used as maximum for all retryable
+ operations such as the getting of a cell's value, starting a row update,
+ etc. Retry interval is a rough function based on hbase.client.pause. At
+ first we retry at this interval but then with backoff, we pretty quickly reach
+ retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
+ ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1d9040a..1175fff 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -18,7 +18,12 @@
<property>
<name>dfs.block.access.token.enable</name>
- <value>true</value>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>dfs.block.size</name>
+ <value>34217472</value>
</property>
<property>
@@ -42,21 +47,11 @@
</property>
<property>
- <name>dfs.client.retry.policy.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<name>dfs.cluster.administrators</name>
<value>hdfs</value>
</property>
<property>
- <name>dfs.content-summary.limit</name>
- <value>5000</value>
- </property>
-
- <property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:50010</value>
</property>
@@ -69,6 +64,7 @@
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/hdfs/data</value>
+ <final>true</final>
</property>
<property>
@@ -84,6 +80,7 @@
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>0</value>
+ <final>true</final>
</property>
<property>
@@ -107,18 +104,13 @@
</property>
<property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.encrypt.data.transfer.cipher.suites</name>
- <value>AES/CTR/NoPadding</value>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>1024</value>
</property>
<property>
- <name>dfs.encryption.key.provider.uri</name>
- <value></value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
<property>
@@ -158,12 +150,7 @@
<property>
<name>dfs.namenode.accesstime.precision</name>
- <value>0</value>
- </property>
-
- <property>
- <name>dfs.namenode.audit.log.async</name>
- <value>true</value>
+ <value>3600000</value>
</property>
<property>
@@ -197,11 +184,6 @@
</property>
<property>
- <name>dfs.namenode.fslock.fair</name>
- <value>false</value>
- </property>
-
- <property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>
@@ -209,6 +191,7 @@
<property>
<name>dfs.namenode.http-address</name>
<value>sandbox.hortonworks.com:50070</value>
+ <final>true</final>
</property>
<property>
@@ -217,13 +200,9 @@
</property>
<property>
- <name>dfs.namenode.inode.attributes.provider.class</name>
- <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
- </property>
-
- <property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/hdfs/namenode</value>
+ <final>true</final>
</property>
<property>
@@ -232,13 +211,8 @@
</property>
<property>
- <name>dfs.namenode.rpc-address</name>
- <value>sandbox.hortonworks.com:8020</value>
- </property>
-
- <property>
<name>dfs.namenode.safemode.threshold-pct</name>
- <value>0.999</value>
+ <value>1.0f</value>
</property>
<property>
@@ -262,6 +236,16 @@
</property>
<property>
+ <name>dfs.nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ </property>
+
+ <property>
+ <name>dfs.nfs3.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ </property>
+
+ <property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
@@ -273,7 +257,7 @@
<property>
<name>dfs.replication</name>
- <value>3</value>
+ <value>1</value>
</property>
<property>
@@ -284,11 +268,13 @@
<property>
<name>dfs.support.append</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
@@ -296,14 +282,4 @@
<value>022</value>
</property>
- <property>
- <name>nfs.exports.allowed.hosts</name>
- <value>* rw</value>
- </property>
-
- <property>
- <name>nfs.file.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
- </property>
-
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index a8c210e..1e78107 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -22,46 +22,11 @@
</property>
<property>
- <name>atlas.cluster.name</name>
- <value>Sandbox</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.maxThreads</name>
- <value>1</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.minThreads</name>
- <value>1</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.synchronous</name>
- <value>true</value>
- </property>
-
- <property>
- <name>atlas.rest.address</name>
- <value>http://sandbox.hortonworks.com:21000</value>
- </property>
-
- <property>
- <name>datanucleus.autoCreateSchema</name>
- <value>false</value>
- </property>
-
- <property>
<name>datanucleus.cache.level2.type</name>
<value>none</value>
</property>
<property>
- <name>datanucleus.fixedDatastore</name>
- <value>true</value>
- </property>
-
- <property>
<name>hive.auto.convert.join</name>
<value>true</value>
</property>
@@ -73,7 +38,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>357913941</value>
+ <value>1000000000</value>
</property>
<property>
@@ -162,16 +127,6 @@
</property>
<property>
- <name>hive.default.fileformat</name>
- <value>TextFile</value>
- </property>
-
- <property>
- <name>hive.default.fileformat.managed</name>
- <value>TextFile</value>
- </property>
-
- <property>
<name>hive.enforce.bucketing</name>
<value>true</value>
</property>
@@ -207,6 +162,11 @@
</property>
<property>
+ <name>hive.exec.failure.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
<name>hive.exec.max.created.files</name>
<value>100000</value>
</property>
@@ -237,11 +197,6 @@
</property>
<property>
- <name>hive.exec.orc.encoding.strategy</name>
- <value>SPEED</value>
- </property>
-
- <property>
<name>hive.exec.parallel</name>
<value>false</value>
</property>
@@ -252,6 +207,16 @@
</property>
<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>67108864</value>
</property>
@@ -297,6 +262,11 @@
</property>
<property>
+ <name>hive.heapsize</name>
+ <value>250</value>
+ </property>
+
+ <property>
<name>hive.limit.optimize.enable</name>
<value>true</value>
</property>
@@ -508,7 +478,7 @@
<property>
<name>hive.prewarm.numcontainers</name>
- <value>3</value>
+ <value>10</value>
</property>
<property>
@@ -518,7 +488,7 @@
<property>
<name>hive.security.authorization.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -538,7 +508,7 @@
<property>
<name>hive.security.metastore.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value>
+ <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
</property>
<property>
@@ -563,7 +533,12 @@
<property>
<name>hive.server2.enable.doAs</name>
- <value>false</value>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.impersonation</name>
+ <value>true</value>
</property>
<property>
@@ -573,7 +548,7 @@
<property>
<name>hive.server2.logging.operation.log.location</name>
- <value>/tmp/hive/operation_logs</value>
+ <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
</property>
<property>
@@ -678,7 +653,7 @@
<property>
<name>hive.tez.container.size</name>
- <value>1024</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index be470f9..e90f594 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>64</value>
+ <value>128</value>
</property>
<property>
@@ -27,13 +27,13 @@
</property>
<property>
- <name>mapred.job.map.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.map.memory.mb</name>
+ <value>512</value>
</property>
<property>
- <name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>512</value>
</property>
<property>
@@ -48,9 +48,7 @@
<property>
<name>mapreduce.admin.user.env</name>
- <value>
- LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64
- </value>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
</property>
<property>
@@ -60,9 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>
- $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure
- </value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,18 +77,14 @@
</property>
<property>
- <name>mapreduce.job.counters.max</name>
- <value>130</value>
- </property>
-
- <property>
<name>mapreduce.job.emit-timeline-data</name>
<value>false</value>
</property>
+ <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
+ <value>1</value>
</property>
<property>
@@ -116,28 +108,13 @@
</property>
<property>
- <name>mapreduce.jobhistory.recovery.enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.recovery.store.class</name>
- <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
- <value>/hadoop/mapreduce/jhs</value>
- </property>
-
- <property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>sandbox.hortonworks.com:19888</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx1228m</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -147,7 +124,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>1536</value>
+ <value>512</value>
</property>
<property>
@@ -182,7 +159,7 @@
<property>
<name>mapreduce.reduce.java.opts</name>
- <value>-Xmx1638m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -192,7 +169,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>2048</value>
+ <value>512</value>
</property>
<property>
@@ -242,7 +219,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>859</value>
+ <value>128</value>
</property>
<property>
@@ -257,7 +234,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx819m -Dhdp.version=${hdp.version}</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -267,7 +244,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>1024</value>
+ <value>512</value>
</property>
<property>
@@ -275,4 +252,4 @@
<value>/user</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index ebdf44a..8256158 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -18,7 +18,7 @@
<property>
<name>hadoop.registry.rm.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -28,29 +28,22 @@
<property>
<name>yarn.acl.enable</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
<name>yarn.admin.acl</name>
- <value>*</value>
+ <value></value>
</property>
<property>
<name>yarn.application.classpath</name>
- <value>
- $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*
- </value>
- </property>
-
- <property>
- <name>yarn.authorization-provider</name>
- <value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
</property>
<property>
<name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>120000</value>
+ <value>60000</value>
</property>
<property>
@@ -79,11 +72,6 @@
</property>
<property>
- <name>yarn.node-labels.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.node-labels.fs-store.retry-policy-spec</name>
<value>2000, 500</value>
</property>
@@ -94,6 +82,11 @@
</property>
<property>
+ <name>yarn.node-labels.manager-class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
+ </property>
+
+ <property>
<name>yarn.nodemanager.address</name>
<value>0.0.0.0:45454</value>
</property>
@@ -105,7 +98,7 @@
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle,spark_shuffle</value>
+ <value>mapreduce_shuffle</value>
</property>
<property>
@@ -114,11 +107,6 @@
</property>
<property>
- <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
- <value>org.apache.spark.network.yarn.YarnShuffleService</value>
- </property>
-
- <property>
<name>yarn.nodemanager.bind-host</name>
<value>0.0.0.0</value>
</property>
@@ -160,7 +148,7 @@
<property>
<name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>120000</value>
+ <value>60000</value>
</property>
<property>
@@ -255,12 +243,12 @@
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
- <value>7168</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>80</value>
+ <value>100</value>
</property>
<property>
@@ -349,11 +337,6 @@
</property>
<property>
- <name>yarn.resourcemanager.scheduler.monitor.enable</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.resourcemanager.state-store.max-completed-applications</name>
<value>${yarn.resourcemanager.max-completed-applications}</value>
</property>
@@ -385,7 +368,7 @@
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8090</value>
+ <value>localhost:8090</value>
</property>
<property>
@@ -425,7 +408,7 @@
<property>
<name>yarn.resourcemanager.zk-address</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <value>localhost:2181</value>
</property>
<property>
@@ -450,22 +433,12 @@
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
- <value>7168</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-vcores</name>
- <value>3</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-vcores</name>
- <value>1</value>
+ <value>1536</value>
</property>
<property>
@@ -494,41 +467,6 @@
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
- <value>/ats/active/</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
- <value>3600</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
- <value>/ats/done/</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
- <value>org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
- <value>60</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
- <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
- </property>
-
- <property>
<name>yarn.timeline-service.generic-application-history.store-class</name>
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
@@ -544,11 +482,6 @@
</property>
<property>
- <name>yarn.timeline-service.leveldb-state-store.path</name>
- <value>/hadoop/yarn/timeline</value>
- </property>
-
- <property>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
<value>/hadoop/yarn/timeline</value>
</property>
@@ -574,23 +507,8 @@
</property>
<property>
- <name>yarn.timeline-service.plugin.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.recovery.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.state-store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
- </property>
-
- <property>
<name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
@@ -604,11 +522,6 @@
</property>
<property>
- <name>yarn.timeline-service.version</name>
- <value>1.5</value>
- </property>
-
- <property>
<name>yarn.timeline-service.webapp.address</name>
<value>sandbox.hortonworks.com:8188</value>
</property>
@@ -618,4 +531,4 @@
<value>sandbox.hortonworks.com:8190</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index df6c45e..d819098 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -32,9 +32,11 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,7 +60,6 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -95,10 +96,10 @@ public class BuildCubeWithEngine {
logger.error("error", e);
exitCode = 1;
}
-
+
long millis = System.currentTimeMillis() - start;
System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName());
-
+
System.exit(exitCode);
}
@@ -359,10 +360,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
- String tableName = segment.getStorageLocationIdentifier();
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+ String tableName = segment.getStorageLocationIdentifier();
+ try (HTable table = new HTable(conf, tableName)) {
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30506e5..486e1e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,15 +46,15 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0</hadoop2.version>
+ <yarn.version>2.6.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>0.14.0</hive.version>
+ <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
<!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
@@ -72,7 +72,7 @@
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.7.1</curator.version>
+ <curator.version>2.6.0</curator.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
<jsch.version>0.1.53</jsch.version>
@@ -844,11 +844,6 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
-
- <repository>
- <id>cloudera</id>
- <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
- </repository>
</repositories>
<build>
@@ -1202,106 +1197,6 @@
</build>
</profile>
<profile>
- <id>cdh5.7</id>
- <properties>
- <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
- <yarn.version>2.6.0-cdh5.7.0</yarn.version>
- <hive.version>1.1.0-cdh5.7.0</hive.version>
- <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
- <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
- <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <fork>true</fork>
- <meminitial>1024m</meminitial>
- <maxmem>2048m</maxmem>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-jamm</id>
- <goals>
- <goal>copy</goal>
- </goals>
- <phase>generate-test-resources</phase>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>com.github.jbellis</groupId>
- <artifactId>jamm</artifactId>
- <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
- <destFileName>jamm.jar</destFileName>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <configuration>
- <append>true</append>
- <destFile>
- ${sonar.jacoco.reportPath}
- </destFile>
- </configuration>
- <executions>
- <execution>
- <id>pre-test</id>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- <configuration>
- <propertyName>surefireArgLine</propertyName>
- </configuration>
- </execution>
- <execution>
- <id>post-test</id>
- <phase>test</phase>
- <goals>
- <goal>report</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.19.1</version>
- <configuration>
- <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
- <excludes>
- <exclude>**/IT*.java</exclude>
- </excludes>
- <systemProperties>
- <property>
- <name>buildCubeUsingProvidedData</name>
- <value>false</value>
- </property>
- <property>
- <name>log4j.configuration</name>
- <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
- </property>
- </systemProperties>
- <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<!-- This profile adds/overrides few features of the 'apache-release'
profile in the parent pom. -->
<id>apache-release</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 8095bf8..ea68855 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- Table getTable(String tableName) throws IOException;
+ HTableInterface getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index cc76b87..d9326f5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private Table mockedAclTable;
- private Table mockedUserTable;
+ private HTableInterface mockedAclTable;
+ private HTableInterface mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public Table getTable(String tableName) throws IOException {
+ public HTableInterface getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index 972eea9..d0aa0ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements Table {
+public class MockHTable implements HTableInterface {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,6 +114,14 @@ public class MockHTable implements Table {
this.columnFamilies.add(columnFamily);
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public byte[] getTableName() {
+ return tableName.getBytes();
+ }
+
@Override
public TableName getName() {
return null;
@@ -192,8 +200,8 @@ public class MockHTable implements Table {
}
@Override
- public boolean[] existsAll(List<Get> list) throws IOException {
- return new boolean[0];
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ return new Boolean[0];
}
/**
@@ -298,6 +306,15 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ // FIXME: implement
+ return null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -429,7 +446,7 @@ public class MockHTable implements Table {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow();
+ filter.filterRow(nkvs);
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -518,11 +535,6 @@ public class MockHTable implements Table {
return false;
}
- @Override
- public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
- return false;
- }
-
/**
* {@inheritDoc}
*/
@@ -543,7 +555,7 @@ public class MockHTable implements Table {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDelete()) {
+ if (kv.isDeleteFamily()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -580,11 +592,6 @@ public class MockHTable implements Table {
return false;
}
- @Override
- public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
- return false;
- }
-
/**
* {@inheritDoc}
*/
@@ -598,7 +605,7 @@ public class MockHTable implements Table {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, null);
+ return incrementColumnValue(row, family, qualifier, amount, true);
}
@Override
@@ -610,6 +617,37 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
+ if (check(row, family, qualifier, null)) {
+ Put put = new Put(row);
+ put.add(family, qualifier, Bytes.toBytes(amount));
+ put(put);
+ return amount;
+ }
+ long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
+ data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
+ return newValue;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isAutoFlush() {
+ return true;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void flushCommits() throws IOException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public void close() throws IOException {
}
@@ -635,6 +673,29 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public void setAutoFlush(boolean autoFlush) {
+ throw new NotImplementedException();
+
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ throw new NotImplementedException();
+
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ throw new NotImplementedException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index d1a1384..1d520c4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,8 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -59,11 +58,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public Table getTable(String tableName) throws IOException {
+ public HTableInterface getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
+ return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ return HBaseConnection.get(hbaseUrl).getTable(userTableName);
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 3e3efec..d693a67 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- Table htable = null;
+ HTableInterface htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,16 +226,17 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
+ htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -249,7 +250,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -265,6 +266,7 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
+ htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -282,7 +284,7 @@ public class AclService implements MutableAclService {
throw e;
}
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -293,16 +295,17 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
+ htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a28cedb..76fab52 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -27,7 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -393,24 +395,33 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
- Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
+
+ Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+ HTable table = null;
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ try {
+ table = new HTable(hconf, tableName);
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- regionCount = sizeMap.size();
+ for (long s : sizeMap.values()) {
+ tableSize += s;
+ }
+
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 6a38638..8972456 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -48,11 +48,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -163,13 +163,14 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- Table htable = null;
+ HTableInterface htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
- put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
+ htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -195,13 +196,14 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- Table htable = null;
+ HTableInterface htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
- put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
+ htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -213,12 +215,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- Table htable = null;
+ HTableInterface htable = null;
try {
- org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
+ HConnection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = conn.getTable(userTableName);
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index ab54882..07c7c6f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
-
- put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+ put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
+ htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,13 +163,14 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
+ htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -184,7 +185,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -215,7 +216,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- Table htable = null;
+ HTableInterface htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..2351412 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -39,9 +39,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -63,7 +63,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
+ private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -74,7 +74,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (Connection conn : connPool.values()) {
+ for (HConnection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -143,7 +143,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -207,9 +207,9 @@ public class HBaseConnection {
// ============================================================================
- // returned Connection can be shared by multiple threads and does not require close()
+ // returned HConnection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static Connection get(String url) {
+ public static HConnection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -217,13 +217,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- Connection connection = connPool.get(url);
+ HConnection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = ConnectionFactory.createConnection(conf);
+ connection = HConnectionManager.createConnection(conf);
connPool.put(url, connection);
}
@@ -242,8 +242,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ public static boolean tableExists(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -263,18 +263,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
- Admin hbase = conn.getAdmin();
- TableName tableName = TableName.valueOf(table);
+ public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
+
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(tableName, newFamilyDescriptor(family));
+ hbase.addColumn(table, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -327,8 +327,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ public static void deleteTable(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
if (!tableExists(conn, tableName)) {
@@ -338,10 +338,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
- hbase.disableTable(TableName.valueOf(tableName));
+ if (hbase.isTableEnabled(tableName)) {
+ hbase.disableTable(tableName);
}
- hbase.deleteTable(TableName.valueOf(tableName));
+ hbase.deleteTable(tableName);
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 74ab017..a44de4f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,15 +30,14 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +68,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- Connection getConnection() throws IOException {
+ HConnection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -121,7 +120,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,12 +237,13 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
+ table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,6 +264,8 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
+ table.flushCommits();
+
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -272,7 +274,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -285,6 +287,7 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
+ table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -305,7 +308,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -314,7 +317,7 @@ public class HBaseResourceStore extends ResourceStore {
}
- private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -333,7 +336,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
@@ -358,7 +361,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -366,8 +369,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.addColumn(B_FAMILY, B_COLUMN, content);
- put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.add(B_FAMILY, B_COLUMN, content);
+ put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
[03/48] kylin git commit: KYLIN_2464,
replace ConcurrentHashMap with ConcurrentMap
Posted by li...@apache.org.
KYLIN_2464, replace ConcurrentHashMap with ConcurrentMap
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/693c6faf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/693c6faf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/693c6faf
Branch: refs/heads/master-hbase0.98
Commit: 693c6faf2f38d19816c8e0c9e0a48c6caaaf18cd
Parents: 6adb73d
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Feb 24 18:33:48 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Feb 24 18:46:44 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceStore.java | 4 ++--
.../common/util/MemoryBudgetController.java | 3 ++-
.../org/apache/kylin/cube/CubeDescManager.java | 16 +++++++-------
.../java/org/apache/kylin/cube/CubeManager.java | 3 ++-
.../apache/kylin/dict/DictionaryManager.java | 3 ++-
.../kylin/dict/lookup/SnapshotManager.java | 3 ++-
.../org/apache/kylin/job/dao/ExecutableDao.java | 3 ++-
.../kylin/job/execution/ExecutableManager.java | 22 +++++++++++---------
.../impl/threadpool/DistributedScheduler.java | 3 ++-
.../apache/kylin/metadata/MetadataManager.java | 3 ++-
.../badquery/BadQueryHistoryManager.java | 3 ++-
.../kylin/metadata/cachesync/Broadcaster.java | 3 ++-
.../kylin/metadata/project/ProjectManager.java | 13 ++++++------
.../realization/RealizationRegistry.java | 3 ++-
.../metadata/streaming/StreamingManager.java | 3 ++-
.../kylin/storage/hybrid/HybridManager.java | 3 ++-
.../kylin/source/kafka/KafkaConfigManager.java | 9 ++++----
17 files changed, 58 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 77143b0..b0e06f5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -30,6 +30,7 @@ import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -63,10 +64,9 @@ abstract public class ResourceStore {
public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
public static final String BAD_QUERY_RESOURCE_ROOT = "/bad_query";
-
protected static final String DEFAULT_STORE_NAME = "kylin_metadata";
- private static final ConcurrentHashMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
+ private static final ConcurrentMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
private static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index ade929c..7a0b919 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -19,6 +19,7 @@
package org.apache.kylin.common.util;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantLock;
import org.slf4j.Logger;
@@ -64,7 +65,7 @@ public class MemoryBudgetController {
// all budget numbers are in MB
private final int totalBudgetMB;
- private final ConcurrentHashMap<MemoryConsumer, ConsumerEntry> booking = new ConcurrentHashMap<MemoryConsumer, ConsumerEntry>();
+ private final ConcurrentMap<MemoryConsumer, ConsumerEntry> booking = new ConcurrentHashMap<MemoryConsumer, ConsumerEntry>();
private int totalReservedMB;
private final ReentrantLock lock = new ReentrantLock();
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 1bf7e97..00fa705 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -63,7 +64,7 @@ public class CubeDescManager {
public static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER = new JsonSerializer<CubeDesc>(CubeDesc.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>();
+ private static final ConcurrentMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>();
public static CubeDescManager getInstance(KylinConfig config) {
CubeDescManager r = CACHE.get(config);
@@ -103,14 +104,14 @@ public class CubeDescManager {
logger.info("Initializing CubeDescManager with config " + config);
this.config = config;
this.cubeDescMap = new CaseInsensitiveStringCache<CubeDesc>(config, "cube_desc");
-
+
// touch lower level metadata before registering my listener
reloadAllCubeDesc();
Broadcaster.getInstance(config).registerListener(new CubeDescSyncListener(), "cube_desc");
}
-
+
private class CubeDescSyncListener extends Broadcaster.Listener {
-
+
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
clearCache();
@@ -122,7 +123,7 @@ public class CubeDescManager {
for (IRealization real : ProjectManager.getInstance(config).listAllRealizations(project)) {
if (real instanceof CubeInstance) {
String descName = ((CubeInstance) real).getDescName();
- reloadCubeDescLocal(descName);
+ reloadCubeDescLocal(descName);
}
}
}
@@ -132,12 +133,12 @@ public class CubeDescManager {
String cubeDescName = cacheKey;
CubeDesc cubeDesc = getCubeDesc(cubeDescName);
String modelName = cubeDesc == null ? null : cubeDesc.getModel().getName();
-
+
if (event == Event.DROP)
removeLocalCubeDesc(cubeDescName);
else
reloadCubeDescLocal(cubeDescName);
-
+
for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(modelName)) {
broadcaster.notifyProjectSchemaUpdate(prj.getName());
}
@@ -237,7 +238,6 @@ public class CubeDescManager {
return cubeDesc;
}
-
/**
* if there is some change need be applied after getting a cubeDesc from front-end, do it here
* @param cubeDesc
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9670b89..073f516 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -33,6 +33,7 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -89,7 +90,7 @@ public class CubeManager implements IRealizationProvider {
private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeManager>();
+ private static final ConcurrentMap<KylinConfig, CubeManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeManager>();
public static CubeManager getInstance(KylinConfig config) {
CubeManager r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 3ba24cf..427bd14 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -59,7 +60,7 @@ public class DictionaryManager {
private static final DictionaryInfo NONE_INDICATOR = new DictionaryInfo();
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, DictionaryManager> CACHE = new ConcurrentHashMap<KylinConfig, DictionaryManager>();
+ private static final ConcurrentMap<KylinConfig, DictionaryManager> CACHE = new ConcurrentHashMap<KylinConfig, DictionaryManager>();
public static DictionaryManager getInstance(KylinConfig config) {
DictionaryManager r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index b45d017..a912696 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict.lookup;
import java.io.IOException;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -47,7 +48,7 @@ public class SnapshotManager {
private static final Logger logger = LoggerFactory.getLogger(SnapshotManager.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, SnapshotManager> SERVICE_CACHE = new ConcurrentHashMap<KylinConfig, SnapshotManager>();
+ private static final ConcurrentMap<KylinConfig, SnapshotManager> SERVICE_CACHE = new ConcurrentHashMap<KylinConfig, SnapshotManager>();
public static SnapshotManager getInstance(KylinConfig config) {
SnapshotManager r = SERVICE_CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index 96505e6..70799d8 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -43,7 +44,7 @@ public class ExecutableDao {
private static final Serializer<ExecutablePO> JOB_SERIALIZER = new JsonSerializer<ExecutablePO>(ExecutablePO.class);
private static final Serializer<ExecutableOutputPO> JOB_OUTPUT_SERIALIZER = new JsonSerializer<ExecutableOutputPO>(ExecutableOutputPO.class);
private static final Logger logger = LoggerFactory.getLogger(ExecutableDao.class);
- private static final ConcurrentHashMap<KylinConfig, ExecutableDao> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableDao>();
+ private static final ConcurrentMap<KylinConfig, ExecutableDao> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableDao>();
private ResourceStore store;
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 48cedb5..0c86d72 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -18,9 +18,13 @@
package org.apache.kylin.job.execution;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.job.dao.ExecutableDao;
@@ -31,18 +35,16 @@ import org.apache.kylin.job.exception.PersistentException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
*/
public class ExecutableManager {
private static final Logger logger = LoggerFactory.getLogger(ExecutableManager.class);
- private static final ConcurrentHashMap<KylinConfig, ExecutableManager> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableManager>();
+ private static final ConcurrentMap<KylinConfig, ExecutableManager> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableManager>();
private final KylinConfig config;
private final ExecutableDao executableDao;
@@ -347,7 +349,7 @@ public class ExecutableManager {
for (AbstractExecutable task : tasks) {
if (task.getId().compareTo(stepId) >= 0) {
logger.debug("rollback task : " + task);
- updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String>newHashMap(), "");
+ updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
index 84e62d5..1f2e958 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java
@@ -23,6 +23,7 @@ import java.net.UnknownHostException;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -72,7 +73,7 @@ public class DistributedScheduler implements Scheduler<AbstractExecutable>, Conn
private DistributedJobLock jobLock;
private static final Logger logger = LoggerFactory.getLogger(DistributedScheduler.class);
- private static final ConcurrentHashMap<KylinConfig, DistributedScheduler> CACHE = new ConcurrentHashMap<KylinConfig, DistributedScheduler>();
+ private static final ConcurrentMap<KylinConfig, DistributedScheduler> CACHE = new ConcurrentHashMap<KylinConfig, DistributedScheduler>();
//keep all segments having running job
private final Set<String> segmentWithLocks = new CopyOnWriteArraySet<>();
private volatile boolean initialized = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 49ec96e..9427ace 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -65,7 +66,7 @@ public class MetadataManager {
public static final Serializer<ExternalFilterDesc> EXTERNAL_FILTER_DESC_SERIALIZER = new JsonSerializer<ExternalFilterDesc>(ExternalFilterDesc.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, MetadataManager> CACHE = new ConcurrentHashMap<KylinConfig, MetadataManager>();
+ private static final ConcurrentMap<KylinConfig, MetadataManager> CACHE = new ConcurrentHashMap<KylinConfig, MetadataManager>();
public static MetadataManager getInstance(KylinConfig config) {
MetadataManager r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
index 86e282e..c7eb133 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.badquery;
import java.io.IOException;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -35,7 +36,7 @@ public class BadQueryHistoryManager {
public static final Serializer<BadQueryHistory> BAD_QUERY_INSTANCE_SERIALIZER = new JsonSerializer<>(BadQueryHistory.class);
private static final Logger logger = LoggerFactory.getLogger(BadQueryHistoryManager.class);
- private static final ConcurrentHashMap<KylinConfig, BadQueryHistoryManager> CACHE = new ConcurrentHashMap<>();
+ private static final ConcurrentMap<KylinConfig, BadQueryHistoryManager> CACHE = new ConcurrentHashMap<>();
private KylinConfig kylinConfig;
private BadQueryHistoryManager(KylinConfig config) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 17b644d..5b45d9e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
@@ -63,7 +64,7 @@ public class Broadcaster {
public static final String SYNC_PRJ_DATA = "project_data"; // the special entity to indicate project data has change, e.g. cube/raw_table update
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, Broadcaster> CACHE = new ConcurrentHashMap<KylinConfig, Broadcaster>();
+ private static final ConcurrentMap<KylinConfig, Broadcaster> CACHE = new ConcurrentHashMap<KylinConfig, Broadcaster>();
public static Broadcaster getInstance(KylinConfig config) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index ca4f7f1..bb1e3ed 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -48,7 +49,7 @@ import com.google.common.collect.Lists;
public class ProjectManager {
private static final Logger logger = LoggerFactory.getLogger(ProjectManager.class);
- private static final ConcurrentHashMap<KylinConfig, ProjectManager> CACHE = new ConcurrentHashMap<KylinConfig, ProjectManager>();
+ private static final ConcurrentMap<KylinConfig, ProjectManager> CACHE = new ConcurrentHashMap<KylinConfig, ProjectManager>();
public static final Serializer<ProjectInstance> PROJECT_SERIALIZER = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
public static ProjectManager getInstance(KylinConfig config) {
@@ -98,7 +99,7 @@ public class ProjectManager {
}
private class ProjectSyncListener extends Broadcaster.Listener {
-
+
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
clearCache();
@@ -107,12 +108,12 @@ public class ProjectManager {
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException {
String project = cacheKey;
-
+
if (event == Event.DROP)
removeProjectLocal(project);
else
reloadProjectLocal(project);
-
+
broadcaster.notifyProjectSchemaUpdate(project);
broadcaster.notifyProjectDataUpdate(project);
}
@@ -249,7 +250,7 @@ public class ProjectManager {
projectMap.remove(norm(proj.getName()));
clearL2Cache();
}
-
+
private void removeProjectLocal(String proj) {
projectMap.remove(norm(proj));
clearL2Cache();
@@ -393,7 +394,7 @@ public class ProjectManager {
}
return projects;
}
-
+
public ExternalFilterDesc getExternalFilterDesc(String project, String extFilter) {
return l2Cache.getExternalFilterDesc(project, extFilter);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
index 77e2679..2d1a4a5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
@@ -37,7 +38,7 @@ import com.google.common.collect.Maps;
public class RealizationRegistry {
private static final Logger logger = LoggerFactory.getLogger(RealizationRegistry.class);
- private static final ConcurrentHashMap<KylinConfig, RealizationRegistry> CACHE = new ConcurrentHashMap<KylinConfig, RealizationRegistry>();
+ private static final ConcurrentMap<KylinConfig, RealizationRegistry> CACHE = new ConcurrentHashMap<KylinConfig, RealizationRegistry>();
public static RealizationRegistry getInstance(KylinConfig config) {
RealizationRegistry r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
index 8cfe87d..48febeb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/streaming/StreamingManager.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -42,7 +43,7 @@ public class StreamingManager {
private static final Logger logger = LoggerFactory.getLogger(StreamingManager.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, StreamingManager> CACHE = new ConcurrentHashMap<KylinConfig, StreamingManager>();
+ private static final ConcurrentMap<KylinConfig, StreamingManager> CACHE = new ConcurrentHashMap<KylinConfig, StreamingManager>();
public static final Serializer<StreamingConfig> STREAMING_SERIALIZER = new JsonSerializer<StreamingConfig>(StreamingConfig.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index 748e873..2d330c0 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -47,7 +48,7 @@ public class HybridManager implements IRealizationProvider {
private static final Logger logger = LoggerFactory.getLogger(HybridManager.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, HybridManager> CACHE = new ConcurrentHashMap<KylinConfig, HybridManager>();
+ private static final ConcurrentMap<KylinConfig, HybridManager> CACHE = new ConcurrentHashMap<KylinConfig, HybridManager>();
public static HybridManager getInstance(KylinConfig config) {
HybridManager r = CACHE.get(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/693c6faf/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
index 775f052..50295c3 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -43,7 +44,7 @@ public class KafkaConfigManager {
private static final Logger logger = LoggerFactory.getLogger(KafkaConfigManager.class);
// static cached instances
- private static final ConcurrentHashMap<KylinConfig, KafkaConfigManager> CACHE = new ConcurrentHashMap<KylinConfig, KafkaConfigManager>();
+ private static final ConcurrentMap<KylinConfig, KafkaConfigManager> CACHE = new ConcurrentHashMap<KylinConfig, KafkaConfigManager>();
private KylinConfig config;
@@ -59,7 +60,7 @@ public class KafkaConfigManager {
private KafkaConfigManager(KylinConfig config) throws IOException {
this.config = config;
this.kafkaMap = new CaseInsensitiveStringCache<KafkaConfig>(config, "kafka");
-
+
// touch lower level metadata before registering my listener
reloadAllKafkaConfig();
Broadcaster.getInstance(config).registerListener(new KafkaSyncListener(), "kafka");
@@ -195,7 +196,7 @@ public class KafkaConfigManager {
throw new IllegalArgumentException("No topic info");
}
- if (kafkaConfig.getKafkaClusterConfigs() == null || kafkaConfig.getKafkaClusterConfigs().size() ==0) {
+ if (kafkaConfig.getKafkaClusterConfigs() == null || kafkaConfig.getKafkaClusterConfigs().size() == 0) {
throw new IllegalArgumentException("No cluster info");
}
@@ -213,7 +214,7 @@ public class KafkaConfigManager {
private void removeKafkaConfigLocal(String name) {
kafkaMap.remove(name);
}
-
+
private void reloadAllKafkaConfig() throws IOException {
ResourceStore store = getStore();
logger.info("Reloading Kafka Metadata from folder " + store.getReadableResourcePath(ResourceStore.KAFKA_RESOURCE_ROOT));
[02/48] kylin git commit: KYLIN-2463,
update and refine CubeMigrationCLI
Posted by li...@apache.org.
KYLIN-2463, update and refine CubeMigrationCLI
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6adb73d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6adb73d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6adb73d3
Branch: refs/heads/master-hbase0.98
Commit: 6adb73d3a286ca008e09cd1384cb4a183bc115ae
Parents: cc1bac9
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Feb 24 13:26:04 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Feb 24 17:22:54 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/tool/CubeMigrationCLI.java | 78 ++++++++++++--------
.../kylin/tool/CubeMigrationCheckCLI.java | 4 +-
2 files changed, 49 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6adb73d3/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index c0042f3..c162a76 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -59,8 +59,10 @@ import org.apache.kylin.dict.lookup.SnapshotTable;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -82,13 +84,13 @@ public class CubeMigrationCLI {
private static final Logger logger = LoggerFactory.getLogger(CubeMigrationCLI.class);
- private static List<Opt> operations;
- private static KylinConfig srcConfig;
- private static KylinConfig dstConfig;
- private static ResourceStore srcStore;
- private static ResourceStore dstStore;
- private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private List<Opt> operations;
+ protected KylinConfig srcConfig;
+ protected KylinConfig dstConfig;
+ private ResourceStore srcStore;
+ private ResourceStore dstStore;
+ private FileSystem hdfsFS;
+ private HBaseAdmin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -96,21 +98,21 @@ public class CubeMigrationCLI {
public static void main(String[] args) throws IOException, InterruptedException {
+ CubeMigrationCLI cli = new CubeMigrationCLI();
if (args.length != 8) {
- usage();
+ cli.usage();
System.exit(1);
}
-
- moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
+ cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
}
- private static void usage() {
+ protected void usage() {
System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot overwriteIfExists realExecute");
System.out.println(" srcKylinConfigUri: The KylinConfig of the cube\u2019s source \n" + "dstKylinConfigUri: The KylinConfig of the cube\u2019s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "purgeOrNot: true or false: whether purge the cube from src server after the migration. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
}
- public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
+ public void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
srcConfig = srcCfg;
srcStore = ResourceStore.getStore(srcConfig);
@@ -160,17 +162,17 @@ public class CubeMigrationCLI {
}
}
- public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
+ public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
}
- public static void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException {
+ public void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException {
CubeMigrationCheckCLI checkCLI = new CubeMigrationCheckCLI(kylinConfig, ifFix);
checkCLI.execute(cubeName);
}
- private static String checkAndGetHbaseUrl() {
+ private String checkAndGetHbaseUrl() {
String srcMetadataUrl = srcConfig.getMetadataUrl();
String dstMetadataUrl = dstConfig.getMetadataUrl();
@@ -192,7 +194,7 @@ public class CubeMigrationCLI {
return srcHbaseUrl.trim();
}
- private static void renameFoldersInHdfs(CubeInstance cube) {
+ protected void renameFoldersInHdfs(CubeInstance cube) throws IOException {
for (CubeSegment segment : cube.getSegments()) {
String jobUuid = segment.getLastBuildJobID();
@@ -204,17 +206,19 @@ public class CubeMigrationCLI {
}
- private static void changeHtableHost(CubeInstance cube) {
+ protected void changeHtableHost(CubeInstance cube) {
+ if (cube.getDescriptor().getEngineType() != IStorageAware.ID_SHARDED_HBASE)
+ return;
for (CubeSegment segment : cube.getSegments()) {
operations.add(new Opt(OptType.CHANGE_HTABLE_HOST, new Object[] { segment.getStorageLocationIdentifier() }));
}
}
- private static void copyACL(CubeInstance cube, String projectName) {
+ private void copyACL(CubeInstance cube, String projectName) {
operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid(), cube.getDescriptor().getModel().getUuid(), projectName }));
}
- private static void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException {
+ private void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException {
List<String> metaItems = new ArrayList<String>();
Set<String> dictAndSnapshot = new HashSet<String>();
@@ -232,19 +236,19 @@ public class CubeMigrationCLI {
}
}
- private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
+ private void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
- throw new IllegalStateException("The target project " + projectName + "does not exist");
+ throw new IllegalStateException("The target project " + projectName + " does not exist");
operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { srcCube, cubeName, projectName }));
}
- private static void purgeAndDisable(String cubeName) throws IOException {
+ private void purgeAndDisable(String cubeName) throws IOException {
operations.add(new Opt(OptType.PURGE_AND_DISABLE, new Object[] { cubeName }));
}
- private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException {
+ protected void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException {
CubeDesc cubeDesc = cube.getDescriptor();
metaResource.add(cube.getResourcePath());
@@ -253,6 +257,7 @@ public class CubeMigrationCLI {
for (TableRef tableRef : cubeDesc.getModel().getAllTables()) {
metaResource.add(TableDesc.concatResourcePath(tableRef.getTableIdentity()));
+ metaResource.add(TableExtDesc.concatResourcePath(tableRef.getTableIdentity()));
}
for (CubeSegment segment : cube.getSegments()) {
@@ -262,11 +267,15 @@ public class CubeMigrationCLI {
}
}
- private static enum OptType {
+ protected enum OptType {
COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, COPY_ACL, PURGE_AND_DISABLE
}
- private static class Opt {
+ protected void addOpt(OptType type, Object[] params) {
+ operations.add(new Opt(type, params));
+ }
+
+ private class Opt {
private OptType type;
private Object[] params;
@@ -285,17 +294,17 @@ public class CubeMigrationCLI {
}
- private static void showOpts() {
+ private void showOpts() {
for (int i = 0; i < operations.size(); ++i) {
showOpt(operations.get(i));
}
}
- private static void showOpt(Opt opt) {
+ private void showOpt(Opt opt) {
logger.info("Operation: " + opt.toString());
}
- private static void doOpts() throws IOException, InterruptedException {
+ private void doOpts() throws IOException, InterruptedException {
int index = 0;
try {
for (; index < operations.size(); ++index) {
@@ -320,12 +329,13 @@ public class CubeMigrationCLI {
}
@SuppressWarnings("checkstyle:methodlength")
- private static void doOpt(Opt opt) throws IOException, InterruptedException {
+ private void doOpt(Opt opt) throws IOException, InterruptedException {
logger.info("Executing operation: " + opt.toString());
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
String tableName = (String) opt.params[0];
+ System.out.println("CHANGE_HTABLE_HOST, table name: " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
@@ -432,6 +442,10 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
+ for (TableRef tableRef : srcCube.getModel().getAllTables()) {
+ project.addTable(tableRef.getTableIdentity());
+ }
+
project.addModel(modelName);
project.removeRealization(RealizationType.CUBE, cubeName);
project.addRealizationEntry(RealizationType.CUBE, cubeName);
@@ -497,7 +511,7 @@ public class CubeMigrationCLI {
}
}
- private static void undo(Opt opt) throws IOException, InterruptedException {
+ private void undo(Opt opt) throws IOException, InterruptedException {
logger.info("Undo operation: " + opt.toString());
switch (opt.type) {
@@ -559,7 +573,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config) {
+ private void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
@@ -572,7 +586,7 @@ public class CubeMigrationCLI {
}
}
- private static void renameHDFSPath(String srcPath, String dstPath) throws IOException, InterruptedException {
+ private void renameHDFSPath(String srcPath, String dstPath) throws IOException, InterruptedException {
int nRetry = 0;
int sleepTime = 5000;
while (!hdfsFS.rename(new Path(srcPath), new Path(dstPath))) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6adb73d3/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
index fe348ba..54fbbc0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -139,7 +140,8 @@ public class CubeMigrationCheckCLI {
List<String> segFullNameList = Lists.newArrayList();
CubeInstance cube = CubeManager.getInstance(dstCfg).getCube(cubeName);
- addHTableNamesForCube(cube, segFullNameList);
+ if (cube.getDescriptor().getStorageType() == IStorageAware.ID_SHARDED_HBASE)
+ addHTableNamesForCube(cube, segFullNameList);
check(segFullNameList);
}
[08/48] kylin git commit: KYLIN-2302 push the value in
statement.setMaxRows(10) to storage
Posted by li...@apache.org.
KYLIN-2302 push the value in statement.setMaxRows(10) to storage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cda4df41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cda4df41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cda4df41
Branch: refs/heads/master-hbase0.98
Commit: cda4df4106551f17311fde737d349af91790b3da
Parents: e875e87
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Feb 28 18:18:36 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Feb 28 18:18:36 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/StorageContext.java | 16 +++++++++++-----
.../apache/kylin/query/relnode/OLAPLimitRel.java | 2 ++
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cda4df41/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 4713d71..a3a851f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.realization.IRealization;
import org.slf4j.Logger;
@@ -35,6 +36,7 @@ public class StorageContext {
private String connUrl;
private int limit = Integer.MAX_VALUE;
+ private boolean overlookOuterLimit = false;
private int offset = 0;
private int finalPushDownLimit = Integer.MAX_VALUE;
private boolean hasSort = false;
@@ -62,7 +64,15 @@ public class StorageContext {
}
public int getLimit() {
- return limit;
+ if (overlookOuterLimit) {
+ return limit;
+ } else {
+ return Math.min(limit, BackdoorToggles.getStatementMaxRows());
+ }
+ }
+
+ public void setOverlookOuterLimit() {
+ this.overlookOuterLimit = true;
}
public void setLimit(int l) {
@@ -81,10 +91,6 @@ public class StorageContext {
this.offset = offset;
}
- public void enableLimit() {
- this.limitEnabled = true;
- }
-
public boolean isLimitEnabled() {
return this.limitEnabled;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cda4df41/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 8179807..2a248cc 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -91,6 +91,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
}
context.afterLimit = true;
+ } else {
+ this.context.storageContext.setOverlookOuterLimit();
}
}
[28/48] kylin git commit: KYLIN-2494 ensure model has no dup among
dimensions and measures
Posted by li...@apache.org.
KYLIN-2494 ensure model has no dup among dimensions and measures
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182ed009
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182ed009
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182ed009
Branch: refs/heads/master-hbase0.98
Commit: 182ed009400629dee1bd7ad0121224cacfd07668
Parents: 2dfffde
Author: Li Yang <li...@apache.org>
Authored: Thu Mar 9 18:03:26 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Mar 9 18:03:26 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ----
.../kylin/metadata/model/DataModelDesc.java | 22 +++++++++-----------
.../kylin/metadata/model/DataModelDescTest.java | 11 ++++++++++
.../test_kylin_inner_join_model_desc.json | 1 +
4 files changed, 22 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 02b2d4b..ad02fe2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -329,10 +329,6 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.cube.allow-appear-in-multiple-projects", "false"));
}
- public boolean allowModelDimensionMetricsOverlap() {
- return Boolean.parseBoolean(getOptional("kylin.model.allow-dimension-metrics-overlap", "false"));
- }
-
// ============================================================================
// JOB
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 1c85f09..0a303ec 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -453,23 +453,21 @@ public class DataModelDesc extends RootPersistentEntity {
}
private boolean validate() {
+
+ // ensure no dup between dimensions/metrics
+ for (ModelDimensionDesc dim : dimensions) {
+ String table = dim.getTable();
+ for (String c : dim.getColumns()) {
+ TblColRef dcol = findColumn(table, c);
+ metrics = ArrayUtils.removeElement(metrics, dcol.getIdentity());
+ }
+ }
+
Set<TblColRef> mcols = new HashSet<>();
for (String m : metrics) {
mcols.add(findColumn(m));
}
- if (!KylinConfig.getInstanceFromEnv().allowModelDimensionMetricsOverlap()) {
- // validate no dup between dimensions/metrics
- for (ModelDimensionDesc dim : dimensions) {
- String table = dim.getTable();
- for (String c : dim.getColumns()) {
- TblColRef dcol = findColumn(table, c);
- if (mcols.contains(dcol))
- throw new IllegalStateException(dcol + " cannot be both dimension and metrics at the same time in " + this);
- }
- }
- }
-
// validate PK/FK are in dimensions
boolean pkfkDimAmended = false;
for (Chain chain : joinsTree.tableChains.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
index 0cfa1f4..b319423 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
@@ -19,7 +19,9 @@
package org.apache.kylin.metadata.model;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.MetadataManager;
@@ -39,6 +41,15 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
public void after() throws Exception {
this.cleanupTestMetadata();
}
+
+ @Test
+ public void testNoDupColInDimAndMeasure() {
+ DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");
+ String[] metrics = model.getMetrics();
+ TblColRef col = model.findColumn("edw.test_cal_dt.cal_dt");
+ assertTrue(metrics.length == 2);
+ assertTrue(ArrayUtils.contains(metrics, col.getIdentity()) == false);
+ }
@Test
public void testGetCopyOf() throws JsonProcessingException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 49758dd..dbb6df8 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -105,6 +105,7 @@
}
],
"metrics": [
+ "edw.test_cal_dt.cal_dt",
"PRICE",
"ITEM_COUNT"
],
[47/48] kylin git commit: KYLIN-2307 Create a branch for master with
HBase 0.98 API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b12173d..51b21b3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,13 +26,12 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -87,13 +86,14 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final BufferedMutator table;
+ final HTableInterface table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getBufferedMutator(htableName);
+ HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getTable(htableName);
+ table.setAutoFlush(false, true);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.mutate(put);
+ table.put(put);
}
@Override
public void close() throws IOException {
- table.flush();
+ table.flushCommits();
table.close();
}
}
class Reader implements IGTScanner {
- final Table table;
+ final HTableInterface table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 82b67b6..1cd6694 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,9 +26,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -52,10 +51,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +116,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -172,7 +171,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicReference<RuntimeException> regionErrorHolder = new AtomicReference<>();
try {
- Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
+ HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 33f8d90..a24d517 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -25,12 +25,11 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -132,8 +131,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
+ HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index cde127e..1e550f9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -195,7 +195,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -234,7 +234,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = (HRegion)env.getRegion();
+ region = env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index feb4842..2814ad6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,8 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -80,8 +79,7 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- Admin admin = conn.getAdmin();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -94,7 +92,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(TableName.valueOf(tableName))) {
+ if (admin.tableExists(tableName)) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -103,7 +101,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -112,7 +110,8 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -127,7 +126,8 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index df3cf08..eacff9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -25,13 +25,13 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,21 +100,19 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Admin admin = null;
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = null;
try {
-
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- admin = conn.getAdmin();
-
+ admin = new HBaseAdmin(conf);
for (String table : oldTables) {
- if (admin.tableExists(TableName.valueOf(table))) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
+ if (admin.tableExists(table)) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(TableName.valueOf(table))) {
- admin.disableTable(TableName.valueOf(table));
+ if (admin.isTableEnabled(table)) {
+ admin.disableTable(table);
}
- admin.deleteTable(TableName.valueOf(table));
+ admin.deleteTable(table);
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 6587d4e..d5b36df 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final Table hTable;
+ private final HTableInterface hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,6 +117,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
+ hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 2f7e164..5b2441c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,20 +69,19 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Admin admin = null;
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = null;
try {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- admin = conn.getAdmin();
-
+ admin = new HBaseAdmin(conf);
for (String table : oldTables) {
- if (admin.tableExists(TableName.valueOf(table))) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
+ if (admin.tableExists(table)) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(TableName.valueOf(table))) {
- admin.disableTable(TableName.valueOf(table));
+ if (admin.isTableEnabled(table)) {
+ admin.disableTable(table);
}
- admin.deleteTable(TableName.valueOf(table));
+ admin.deleteTable(table);
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 56f867a..a150607 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,11 +21,9 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -40,8 +38,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -52,7 +50,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
+ hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 581de38..68c0a39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -26,19 +26,19 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -89,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static Admin hbaseAdmin;
+ private static HBaseAdmin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -134,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
- hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ hbaseAdmin = new HBaseAdmin(conf);
hdfsFS = HadoopUtil.getWorkingFileSystem();
@@ -233,7 +233,6 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
-
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -327,8 +326,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- TableName tableName = TableName.valueOf((String) opt.params[0]);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -450,11 +449,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table srcAclHtable = null;
- Table destAclHtable = null;
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -474,6 +473,7 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -504,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- TableName tableName = TableName.valueOf((String) opt.params[0]);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -539,12 +539,13 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- Table destAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -561,7 +562,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config) {
+ private static void updateMeta(KylinConfig config){
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 20d0f7d..8bd4abf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private Admin hbaseAdmin;
+ private HBaseAdmin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,8 +130,9 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ hbaseAdmin = new HBaseAdmin(conf);
+
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -188,10 +189,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
+ hbaseAdmin.disableTable(sepNameList[0]);
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
- hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
+ hbaseAdmin.modifyTable(sepNameList[0], desc);
+ hbaseAdmin.enableTable(sepNameList[0]);
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 29c738e..c9e969f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,8 +44,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -86,8 +85,7 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -205,10 +203,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -229,7 +227,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
- public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -244,7 +242,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -270,15 +268,16 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.modifyTable(tableName, desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(TableName.valueOf(tableName));
+ hbaseAdmin.enableTable(tableName);
return true;
}
- private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+
+ private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -299,12 +298,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final Admin hbaseAdmin;
+ private final HBaseAdmin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -425,7 +424,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 1cdb2f8..61c73d5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -236,9 +235,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table aclHtable = null;
+ HTableInterface aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -258,6 +257,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+ aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index dd5f8fa..86ba22f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- Connection conn = HBaseConnection.get(hbaseUrl);
+ HConnection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(Connection conn) throws IOException {
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ private static void prepareData(HConnection conn) throws IOException {
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 940d64a..6749d6c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,11 +24,9 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -57,8 +55,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -73,12 +71,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
- if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
- hbaseAdmin.disableTable(TableName.valueOf(htableName));
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(TableName.valueOf(htableName));
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 1daca0a..937b65f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -32,15 +31,12 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -62,31 +58,30 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ public HBaseRegionSizeCalculator(HTable table) throws IOException {
+ this(table, new HBaseAdmin(table.getConfiguration()));
+ }
- Table table = null;
- Admin admin = null;
- try {
- table = hbaseConnection.getTable(TableName.valueOf(tableName));
- admin = hbaseConnection.getAdmin();
+ /** Constructor for unit testing */
+ HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ try {
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
+ logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
// Get regions for table.
- RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
- List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
+ Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionLocation hRegionLocation : regionLocationList) {
- tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
+ for (HRegionInfo regionInfo : tableRegionInfos) {
+ tableRegions.add(regionInfo.getRegionName());
}
- ClusterStatus clusterStatus = admin.getClusterStatus();
+ ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -110,7 +105,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(admin);
+ IOUtils.closeQuietly(hBaseAdmin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index a2f60d4..266f7e7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,10 +23,9 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -43,8 +42,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index 8dd2164..1db60fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- Connection conn = getConnection();
- Admin hadmin = conn.getAdmin();
+ HConnection conn = getConnection();
+ HBaseAdmin hadmin = new HBaseAdmin(conn);
try {
- boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
+ boolean tableExist = hadmin.tableExists(tableName);
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -120,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- Connection conn = getConnection();
- Table table = conn.getTable(TableName.valueOf(tableName));
+ HConnection conn = getConnection();
+ HTableInterface table = conn.getTable(tableName);
byte[] key = new byte[8 + 4];//time + id
@@ -136,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.addColumn(CF, QN, cell);
+ put.add(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -172,8 +172,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- Connection conn = getConnection();
- Table table = conn.getTable(TableName.valueOf(tableName));
+ HConnection conn = getConnection();
+ HTableInterface table = conn.getTable(tableName);
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -208,7 +208,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(Table table) throws IOException {
+ private static long getFirstKeyTime(HTableInterface table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -226,8 +226,8 @@ public class HbaseStreamingInput {
}
- private static Connection getConnection() throws IOException {
- return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ private static HConnection getConnection() throws IOException {
+ return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ea05ab2..ca1a060 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,11 +23,10 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -51,8 +50,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index df4e912..8ff5b0f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,14 +30,10 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +52,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+
// get all kylin hbase tables
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -77,13 +73,12 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- TableName tableName = TableName.valueOf(htableName);
- if (hbaseAdmin.tableExists(tableName)) {
- if (hbaseAdmin.isTableEnabled(tableName)) {
- hbaseAdmin.disableTable(tableName);
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(tableName);
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index bba6745..1ea8e8d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,13 +22,12 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -60,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- Connection conn = null;
- Table table = null;
+ HConnection conn = null;
+ HTableInterface table = null;
ResultScanner scanner = null;
try {
- conn = ConnectionFactory.createConnection(hconf);
- table = conn.getTable(TableName.valueOf(hbaseTable));
+ conn = HConnectionManager.createConnection(hconf);
+ table = conn.getTable(hbaseTable);
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index db516bb..01edb1f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,12 +22,11 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -71,8 +70,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- Connection conn = ConnectionFactory.createConnection(conf);
- Table tableInterface = conn.getTable(TableName.valueOf(htableName));
+ HConnection conn = HConnectionManager.createConnection(conf);
+ HTableInterface tableInterface = conn.getTable(htableName);
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f6b65ab..23e7e10 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,9 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -59,7 +57,6 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,8 +77,7 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -157,22 +153,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- Admin hbaseAdmin;
+ HBaseAdmin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
- if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
- hbaseAdmin.disableTable(TableName.valueOf(htableName));
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(TableName.valueOf(htableName));
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index 42a54c8..e36f662 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,18 +24,16 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,15 +49,14 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private Admin hbaseAdmin;
+ private HBaseAdmin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
- hbaseAdmin = conn.getAdmin();
+ this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -169,9 +166,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(TableName.valueOf(tableName));
- hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
- hbaseAdmin.enableTable(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.enableTable(tableName);
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 278c2b8..072ee44 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -60,16 +60,6 @@
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <scope>provided</scope>
- </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index c162a76..8ada323 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -462,11 +462,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table srcAclHtable = null;
- Table destAclHtable = null;
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -486,6 +486,7 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -551,12 +552,13 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- Table destAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1877487f/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table aclHtable = null;
+ HTableInterface aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+ aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[29/48] kylin git commit: minor changes
Posted by li...@apache.org.
minor changes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/181890e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/181890e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/181890e5
Branch: refs/heads/master-hbase0.98
Commit: 181890e542783e66078ad707f37dcb273abae596
Parents: 7eb9969
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 9 21:19:26 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 9 21:19:30 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/QueryContext.java | 3 +++
.../org/apache/kylin/rest/service/QueryService.java | 12 ++++--------
2 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/181890e5/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 5457aa5..0b8d519 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -18,6 +18,7 @@
package org.apache.kylin.common;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -38,6 +39,8 @@ public class QueryContext {
private QueryContext() {
// use QueryContext.current() instead
+
+ queryId = UUID.randomUUID().toString();
}
public static QueryContext current() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/181890e5/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 122b823..6a38638 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -41,7 +41,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
@@ -237,7 +236,7 @@ public class QueryService extends BasicService {
public void logQuery(final SQLRequest request, final SQLResponse response) {
final String user = SecurityContextHolder.getContext().getAuthentication().getName();
- final Set<String> realizationNames = new HashSet<String>();
+ final List<String> realizationNames = new LinkedList<>();
final Set<Long> cuboidIds = new HashSet<Long>();
float duration = response.getDuration() / (float) 1000;
boolean storageCacheUsed = response.isStorageCacheUsed();
@@ -251,8 +250,7 @@ public class QueryService extends BasicService {
}
if (ctx.realization != null) {
- String realizationName = ctx.realization.getName();
- realizationNames.add(realizationName);
+ realizationNames.add(ctx.realization.getCanonicalName());
}
}
@@ -332,10 +330,8 @@ public class QueryService extends BasicService {
BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
final QueryContext queryContext = QueryContext.current();
- final String queryId = UUID.randomUUID().toString();
- queryContext.setQueryId(queryId);
- try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
+ try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId())) {
String sql = sqlRequest.getSql();
String project = sqlRequest.getProject();
logger.info("Using project: " + project);
@@ -431,7 +427,7 @@ public class QueryService extends BasicService {
return response;
}
- private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
+ protected void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
checkAuthorization(sqlResponse.getCube());
}
[35/48] kylin git commit: minor, refine kylin config
Posted by li...@apache.org.
minor, refine kylin config
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/82b22364
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/82b22364
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/82b22364
Branch: refs/heads/master-hbase0.98
Commit: 82b223641047a215a0b212f4c3d77e21d1a5203c
Parents: 0301da0
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Mar 11 17:49:23 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 17:49:23 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 20 +++++++++---
.../apache/kylin/rest/service/AdminService.java | 32 +-------------------
2 files changed, 17 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/82b22364/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 4fcc61f..c6b1511 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -228,7 +228,7 @@ public class KylinConfig extends KylinConfigBase {
Properties conf = new Properties();
try {
OrderedProperties orderedProperties = getKylinOrderedProperties();
- for (Map.Entry<String, String> each: orderedProperties.entrySet()) {
+ for (Map.Entry<String, String> each : orderedProperties.entrySet()) {
conf.put(each.getKey(), each.getValue());
}
} catch (FileNotFoundException e) {
@@ -238,7 +238,7 @@ public class KylinConfig extends KylinConfigBase {
return conf;
}
- public static OrderedProperties getKylinOrderedProperties() throws FileNotFoundException {
+ private static OrderedProperties getKylinOrderedProperties() throws FileNotFoundException {
File propFile = getKylinPropertiesFile();
if (propFile == null || !propFile.exists()) {
logger.error("fail to locate " + KYLIN_CONF_PROPERTIES_FILE);
@@ -270,9 +270,21 @@ public class KylinConfig extends KylinConfigBase {
}
}
- public static String getConfigAsString() throws IOException {
- OrderedProperties orderedProperties = getKylinOrderedProperties();
+ public String getConfigAsString() throws IOException {
+ Properties allProps = getAllProperties();
+ OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties();
+
final StringBuilder sb = new StringBuilder();
+
+ for (Map.Entry<Object, Object> entry : allProps.entrySet()) {
+ String key = entry.getKey().toString();
+ String value = entry.getValue().toString();
+ if (!orderedProperties.containsProperty(key)) {
+ orderedProperties.setProperty(key, value);
+ } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value)) {
+ orderedProperties.setProperty(key, value);
+ }
+ }
for (Map.Entry<String, String> entry : orderedProperties.entrySet()) {
sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/82b22364/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index 4cacb6b..66725dc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -20,7 +20,6 @@ package org.apache.kylin.rest.service;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.lang.reflect.Method;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
@@ -28,7 +27,6 @@ import java.util.TreeMap;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.util.OrderedProperties;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -96,7 +94,7 @@ public class AdminService extends BasicService {
logger.debug("Get Kylin Runtime Config");
try {
- return getAllConfigAsString();
+ return KylinConfig.getInstanceFromEnv().getConfigAsString();
} catch (IOException e) {
throw new InternalErrorException("Failed to get Kylin Runtime Config", e);
}
@@ -113,32 +111,4 @@ public class AdminService extends BasicService {
}
}
- private String getAllConfigAsString() throws IOException {
- Properties allProps;
- try {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- Method getAllMethod = KylinConfigBase.class.getDeclaredMethod("getAllProperties");
- getAllMethod.setAccessible(true);
- allProps = (Properties) getAllMethod.invoke(kylinConfig);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties();
-
- final StringBuilder sb = new StringBuilder();
-
- for (Map.Entry<Object, Object> entry : allProps.entrySet()) {
- String key = entry.getKey().toString();
- String value = entry.getValue().toString();
- if (!orderedProperties.containsProperty(key)) {
- orderedProperties.setProperty(key, value);
- } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value))
- orderedProperties.setProperty(key, value);
- }
- for (Map.Entry<String, String> entry : orderedProperties.entrySet()) {
- sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
- }
- return sb.toString();
- }
}
[41/48] kylin git commit: KYLIN-2497 optimize SelfDefineSortableKey
(by xiefan)
Posted by li...@apache.org.
KYLIN-2497 optimize SelfDefineSortableKey (by xiefan)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/77fe5813
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/77fe5813
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/77fe5813
Branch: refs/heads/master-hbase0.98
Commit: 77fe58136e0a00a64f9149e3d79532d92eeb77aa
Parents: 181b6c5
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Mar 12 10:50:24 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Mar 12 10:50:24 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/KylinReducer.java | 2 +-
.../mr/steps/FactDistinctColumnsMapper.java | 18 ++--
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../engine/mr/steps/SelfDefineSortableKey.java | 106 +++++++++----------
.../mr/steps/NumberDictionaryForestTest.java | 19 ++--
.../mr/steps/SelfDefineSortableKeyTest.java | 82 +++++++++++---
6 files changed, 139 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/77fe5813/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index d428757..e47af9c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -42,7 +42,7 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI
final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
try {
if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Accepting Mapper Key with ordinal: " + reduceCounter);
+ logger.info("Accepting Reducer Key with ordinal: " + reduceCounter);
}
doReduce(key, values, context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/77fe5813/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 07e788c..9f65163 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -31,6 +31,7 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,6 +41,7 @@ import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
+
/**
*/
public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN, Object> {
@@ -50,6 +52,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
BYTES
}
+
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
@@ -68,6 +71,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
private int partitionColumnIndex = -1;
private boolean needFetchPartitionCol = true;
+ private SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+
@Override
protected void setup(Context context) throws IOException {
super.setup(context);
@@ -137,7 +142,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
String[] row = flatTableInputFormat.parseMapperInput(record);
context.getCounter(RawDataCounter.BYTES).increment(countSizeInBytes(row));
-
for (int i = 0; i < factDictCols.size(); i++) {
String fieldValue = row[dictionaryColumnIndex[i]];
if (fieldValue == null)
@@ -161,9 +165,9 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
- sortableKey.setText(outputKey);
+ DataType type = factDictCols.get(i).getType();
+ sortableKey.init(outputKey, type);
//judge type
- sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
context.write(sortableKey, EMPTY_TEXT);
// log a few rows for troubleshooting
@@ -189,8 +193,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
tmpbuf.put(MARK_FOR_PARTITION_COL);
tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
- sortableKey.setText(outputKey);
- sortableKey.setTypeId((byte) 0);
+ sortableKey.init(outputKey, (byte) 0);
context.write(sortableKey, EMPTY_TEXT);
}
}
@@ -247,13 +250,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
hllBuf.clear();
hll.writeRegisters(hllBuf);
outputValue.set(hllBuf.array(), 0, hllBuf.position());
- sortableKey.setText(outputKey);
- sortableKey.setTypeId((byte) 0);
+ sortableKey.init(outputKey, (byte) 0);
context.write(sortableKey, outputValue);
}
}
}
+
private int countNewSize(int oldSize, int dataSize) {
int newSize = oldSize * 2;
while (newSize < dataSize) {
@@ -261,4 +264,5 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
return newSize;
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/77fe5813/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 34624e3..458af69 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -54,7 +54,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends Kyli
protected IMRTableInputFormat flatTableInputFormat;
protected Text outputKey = new Text();
- protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ //protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
protected Text outputValue = new Text();
protected int errorRecordCounter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/77fe5813/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index 74525d3..c75abea 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -17,20 +17,16 @@
*/
package org.apache.kylin.engine.mr.steps;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-/**
- * Created by xiefan on 16-11-1.
- */
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.metadata.datatype.DataType;
+
public class SelfDefineSortableKey implements WritableComparable<SelfDefineSortableKey> {
+
public enum TypeFlag {
NONE_NUMERIC_TYPE,
INTEGER_FAMILY_TYPE,
@@ -39,51 +35,48 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
- private Text text;
+ private Text rawKey;
- private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKey.class);
+ private Object keyInObj;
public SelfDefineSortableKey() {
}
- public SelfDefineSortableKey(byte typeId, Text text) {
+ public SelfDefineSortableKey(Text key, DataType type) {
+ init(key, getTypeIdByDatatype(type));
+ }
+
+ public void init(Text key, byte typeId) {
this.typeId = typeId;
- this.text = text;
+ this.rawKey = key;
+ if (isNumberFamily()) {
+ String valueStr = new String(key.getBytes(), 1, key.getLength() - 1);
+ if (isIntegerFamily()) {
+ this.keyInObj = Long.parseLong(valueStr);
+ } else {
+ this.keyInObj = Double.parseDouble(valueStr);
+ }
+ } else {
+ this.keyInObj = key;
+ }
+ }
+
+
+ public void init(Text key, DataType type) {
+ init(key, getTypeIdByDatatype(type));
}
@Override
public int compareTo(SelfDefineSortableKey o) {
- if (!o.isNumberFamily()) {
- return this.text.compareTo(o.text);
+ if (this.typeId != o.typeId)
+ throw new IllegalStateException("Error. Incompatible types");
+ if (!isNumberFamily()) {
+ return ((Text) this.keyInObj).compareTo(((Text) o.keyInObj));
} else {
- byte[] data1 = this.text.getBytes();
- byte[] data2 = o.text.getBytes();
- String str1 = new String(data1, 1, data1.length - 1);
- String str2 = new String(data2, 1, data2.length - 1);
- if (str1.equals("") || str2.equals("")) {
- //should not achieve here
- logger.error("none numeric value!");
- return 0;
- }
- if (o.isIntegerFamily()) { //integer type
- try {
- Long num1 = Long.parseLong(str1);
- Long num2 = Long.parseLong(str2);
- return num1.compareTo(num2);
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException when parse integer family number.str1:" + str1 + " str2:" + str2);
- e.printStackTrace();
- return 0;
- }
- } else { //other numeric type
- try {
- Double num1 = Double.parseDouble(str1);
- Double num2 = Double.parseDouble(str2);
- return num1.compareTo(num2);
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException when parse doul family number.str1:" + str1 + " str2:" + str2);
- return 0;
- }
+ if (isIntegerFamily()) {
+ return Long.compare((Long) this.keyInObj, (Long) o.keyInObj);
+ } else {
+ return Double.compare((Double) this.keyInObj, (Double) o.keyInObj);
}
}
}
@@ -91,14 +84,15 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeByte(typeId);
- text.write(dataOutput);
+ rawKey.write(dataOutput);
}
@Override
public void readFields(DataInput dataInput) throws IOException {
this.typeId = dataInput.readByte();
- this.text = new Text();
- text.readFields(dataInput);
+ Text inputKey = new Text();
+ inputKey.readFields(dataInput);
+ init(inputKey, typeId);
}
public short getTypeId() {
@@ -106,7 +100,7 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
}
public Text getText() {
- return text;
+ return rawKey;
}
public boolean isNumberFamily() {
@@ -119,25 +113,21 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
}
- public boolean isOtherNumericFamily() {
- return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- }
- public void setTypeIdByDatatype(DataType type) {
+ public byte getTypeIdByDatatype(DataType type) {
if (!type.isNumberFamily()) {
- this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
+ return (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
} else if (type.isIntegerFamily()) {
- this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
+ return (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
} else {
- this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
+ return (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
}
}
-
+
public void setTypeId(byte typeId) {
this.typeId = typeId;
}
- public void setText(Text text) {
- this.text = text;
- }
}
+
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/77fe5813/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index 1908992..c31377c 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -39,7 +39,6 @@ import org.apache.kylin.dict.NumberDictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.TrieDictionaryForest;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
import org.junit.Ignore;
import org.junit.Test;
@@ -49,19 +48,20 @@ import org.junit.Test;
public class NumberDictionaryForestTest {
+
@Test
public void testNumberDictionaryForestLong() {
List<String> list = randomLongData(100);
- testData(list, TypeFlag.INTEGER_FAMILY_TYPE);
+ testData(list, SelfDefineSortableKey.TypeFlag.INTEGER_FAMILY_TYPE);
}
@Test
public void testNumberDictionaryForestDouble() {
List<String> list = randomDoubleData(100);
- testData(list, TypeFlag.DOUBLE_FAMILY_TYPE);
+ testData(list, SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE);
}
- private void testData(List<String> list, TypeFlag flag) {
+ private void testData(List<String> list, SelfDefineSortableKey.TypeFlag flag) {
//stimulate map-reduce job
ArrayList<SelfDefineSortableKey> keyList = createKeyList(list, (byte) flag.ordinal());
Collections.sort(keyList);
@@ -127,7 +127,7 @@ public class NumberDictionaryForestTest {
dict2.dump(System.out);
}
-
+
@Test
public void testMerge() {
// mimic the logic as in MergeCuboidMapper
@@ -136,7 +136,7 @@ public class NumberDictionaryForestTest {
b1.addValue("3");
b1.addValue("23");
TrieDictionaryForest<String> dict1 = b1.build();
-
+
NumberDictionaryForestBuilder b2 = new NumberDictionaryForestBuilder();
b2.addValue("0");
b2.addValue("2");
@@ -144,7 +144,7 @@ public class NumberDictionaryForestTest {
b2.addValue("15");
b2.addValue("23");
TrieDictionaryForest<String> dict2 = b2.build();
-
+
assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId());
assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue());
@@ -167,7 +167,7 @@ public class NumberDictionaryForestTest {
@Ignore
@Test
- public void testDecimalsWithBeginZero(){
+ public void testDecimalsWithBeginZero() {
List<String> testData = new ArrayList<>();
testData.add("000000000000000000000000000.4868");
testData.add("00000000000000000000000000000000000000");
@@ -235,7 +235,8 @@ public class NumberDictionaryForestTest {
//System.out.println("arrays toString:"+Arrays.toString(valueField));
Text outputKey = new Text();
outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey);
+ SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ sortableKey.init(outputKey, typeFlag);
keyList.add(sortableKey);
}
return keyList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/77fe5813/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
index df6287f..551998f 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.mr.steps;
+import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
@@ -29,8 +30,8 @@ import java.util.Random;
import java.util.UUID;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
import org.junit.Test;
/**
@@ -55,7 +56,7 @@ public class SelfDefineSortableKeyTest {
System.out.println("test numbers:" + longList);
ArrayList<String> strNumList = listToStringList(longList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
System.out.println(keyList.get(0).isIntegerFamily());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
@@ -91,8 +92,7 @@ public class SelfDefineSortableKeyTest {
System.out.println("test numbers:" + doubleList);
ArrayList<String> strNumList = listToStringList(doubleList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- System.out.println(keyList.get(0).isOtherNumericFamily());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
for (SelfDefineSortableKey key : keyList) {
@@ -121,8 +121,7 @@ public class SelfDefineSortableKeyTest {
strList.add("hello"); //duplicate
strList.add("123");
strList.add("");
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal());
- System.out.println(keyList.get(0).isOtherNumericFamily());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strList, (byte) SelfDefineSortableKey.TypeFlag.NONE_NUMERIC_TYPE.ordinal());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
for (SelfDefineSortableKey key : keyList) {
@@ -138,6 +137,60 @@ public class SelfDefineSortableKeyTest {
}
@Test
+ public void testPerformance() {
+ Random rand = new Random(System.currentTimeMillis());
+ ArrayList<Double> doubleList = new ArrayList<>();
+ int count = 10 * 10000;
+ for (int i = 0; i < count; i++) {
+ doubleList.add(rand.nextDouble());
+ }
+ doubleList.add(0.0);
+ doubleList.add(0.0); //test duplicate
+ doubleList.add(-1.0); //test negative number
+ doubleList.add(Double.MAX_VALUE);
+ doubleList.add(-Double.MAX_VALUE);
+ //System.out.println(Double.MIN_VALUE);
+
+
+ ArrayList<String> strNumList = listToStringList(doubleList);
+ //System.out.println("test num strs list:"+strNumList);
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+
+ System.out.println("start to test str sort");
+ long start = System.currentTimeMillis();
+ Collections.sort(strNumList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+
+ System.out.println("start to test double sort");
+ start = System.currentTimeMillis();
+ Collections.sort(keyList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+ List<ByteArray> byteList = new ArrayList<>();
+ for (String str : strNumList) {
+ byteList.add(new ByteArray(Bytes.toBytes(str)));
+ }
+ System.out.println("start to test byte array sort");
+ start = System.currentTimeMillis();
+ Collections.sort(byteList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+ //test new key
+ List<SelfDefineSortableKey> newKeyList = new ArrayList<>();
+ for (String str : strNumList) {
+ SelfDefineSortableKey key = new SelfDefineSortableKey();
+ key.init(new Text(str), (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ newKeyList.add(key);
+ }
+ System.out.println("start to test new sortable key");
+ start = System.currentTimeMillis();
+ Collections.sort(newKeyList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+ }
+
+
+ @Test
public void testIllegalNumber() {
Random rand = new Random(System.currentTimeMillis());
ArrayList<Double> doubleList = new ArrayList<>();
@@ -156,18 +209,19 @@ public class SelfDefineSortableKeyTest {
ArrayList<String> strNumList = listToStringList(doubleList);
strNumList.add("fjaeif"); //illegal type
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- System.out.println(keyList.get(0).isOtherNumericFamily());
- Collections.sort(keyList);
- for (SelfDefineSortableKey key : keyList) {
- printKey(key);
+ try {
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ Collections.sort(keyList);
+ fail("Need catch exception");
+ }catch(Exception e){
+ //correct
}
}
@Test
public void testEnum() {
- TypeFlag flag = TypeFlag.DOUBLE_FAMILY_TYPE;
+ SelfDefineSortableKey.TypeFlag flag = SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE;
System.out.println((byte) flag.ordinal());
int t = (byte) flag.ordinal();
System.out.println(t);
@@ -176,7 +230,6 @@ public class SelfDefineSortableKeyTest {
private <T> ArrayList<String> listToStringList(ArrayList<T> list) {
ArrayList<String> strList = new ArrayList<>();
for (T t : list) {
- System.out.println(t.toString());
strList.add(t.toString());
}
return strList;
@@ -193,7 +246,8 @@ public class SelfDefineSortableKeyTest {
Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1);
Text outputKey = new Text();
outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey);
+ SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ sortableKey.init(outputKey, typeFlag);
keyList.add(sortableKey);
}
return keyList;
[18/48] kylin git commit: revert KYLIN-2483
Posted by li...@apache.org.
revert KYLIN-2483
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac35cd39
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac35cd39
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac35cd39
Branch: refs/heads/master-hbase0.98
Commit: ac35cd39a59cca817d9890181d9a3fb8aca23124
Parents: 3fc6843
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Mar 6 09:35:11 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 6 09:35:11 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ----
.../src/main/java/org/apache/kylin/storage/StorageContext.java | 4 +---
2 files changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ac35cd39/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3a87d3d..5d665df 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -816,10 +816,6 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000"));
}
- public int getMergeSortPartitionResultsMinLimit() {
- return Integer.parseInt(getOptional("kylin.query.merge-sort-partition-results.min-limit", "100"));
- }
-
public long getQueryMaxScanBytes() {
long value = Long.parseLong(getOptional("kylin.query.max-scan-bytes", "0"));
return value > 0 ? value : Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ac35cd39/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index f4211ff..998f1db 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.realization.IRealization;
@@ -135,8 +134,7 @@ public class StorageContext {
}
public static boolean mergeSortPartitionResults(int finalPushDownLimit) {
- return isValidPushDownLimit(finalPushDownLimit) && //
- (finalPushDownLimit > KylinConfig.getInstanceFromEnv().getMergeSortPartitionResultsMinLimit());
+ return isValidPushDownLimit(finalPushDownLimit);
}
public long getDeadline() {
[31/48] kylin git commit: minor,
add missing dependency for tool-assembly module
Posted by li...@apache.org.
minor, add missing dependency for tool-assembly module
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70dda404
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70dda404
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70dda404
Branch: refs/heads/master-hbase0.98
Commit: 70dda40448bbd5622788dbfea591bf54aa4cdc8a
Parents: 181890e
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 10 14:16:37 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 10 14:16:48 2017 +0800
----------------------------------------------------------------------
build/bin/diag.sh | 2 +-
tool-assembly/pom.xml | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dda404/build/bin/diag.sh
----------------------------------------------------------------------
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
index 5f3cec3..e9354a2 100644
--- a/build/bin/diag.sh
+++ b/build/bin/diag.sh
@@ -47,7 +47,7 @@ then
fi
mkdir -p ${KYLIN_HOME}/ext
- export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+ export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
if [ ${#patient} -eq 36 ]; then
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dda404/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
index 126c0bc..f7faad0 100644
--- a/tool-assembly/pom.xml
+++ b/tool-assembly/pom.xml
@@ -91,6 +91,7 @@
<artifactSet>
<includes>
<!-- shade the httpcore to avoid the lower version conflict with HBase one -->
+ <include>org.apache.httpcomponents:httpclient</include>
<include>org.apache.httpcomponents:httpcore</include>
<include>org.slf4j:slf4j-api</include>
<include>org.slf4j:slf4j-log4j12</include>
[11/48] kylin git commit: KYLIN-2302 bug fix
Posted by li...@apache.org.
KYLIN-2302 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d571635
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d571635
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d571635
Branch: refs/heads/master-hbase0.98
Commit: 7d571635fdcd74b97e1345465cc0535283a21f95
Parents: aed28ec
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Mar 1 10:21:45 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 1 10:21:54 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/storage/StorageContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d571635/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index a3a851f..21a926b 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -64,7 +64,7 @@ public class StorageContext {
}
public int getLimit() {
- if (overlookOuterLimit) {
+ if (overlookOuterLimit || BackdoorToggles.getStatementMaxRows() == null) {
return limit;
} else {
return Math.min(limit, BackdoorToggles.getStatementMaxRows());
[44/48] kylin git commit: KYLIN-2503 KYLIN-2502 Hive/spark steps
should show YARN app link
Posted by li...@apache.org.
KYLIN-2503 KYLIN-2502 Hive/spark steps should show YARN app link
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47b7c800
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47b7c800
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47b7c800
Branch: refs/heads/master-hbase0.98
Commit: 47b7c800ecdfe14d10426d64eae6b47e11f2a328
Parents: f0eca24
Author: shaofengshi <sh...@apache.org>
Authored: Sun Mar 12 14:13:04 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 12 14:13:04 2017 +0800
----------------------------------------------------------------------
.../kylin/common/util/BufferedLogger.java | 6 +-
.../kylin/job/common/PatternedLogger.java | 130 +++++++++++++++++++
.../kylin/job/common/ShellExecutable.java | 106 +--------------
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../kylin/engine/spark/SparkExecutable.java | 19 +--
.../source/hive/CreateFlatHiveTableStep.java | 5 +-
.../apache/kylin/source/hive/HiveMRInput.java | 6 +-
7 files changed, 153 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/core-common/src/main/java/org/apache/kylin/common/util/BufferedLogger.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BufferedLogger.java b/core-common/src/main/java/org/apache/kylin/common/util/BufferedLogger.java
index cef598d..02a1fa3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BufferedLogger.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BufferedLogger.java
@@ -27,6 +27,8 @@ public class BufferedLogger implements Logger {
private final org.slf4j.Logger wrappedLogger;
private final StringBuilder buffer = new StringBuilder();
+ private static int MAX_BUFFER_SIZE = 10 * 1024 * 1024;
+
public BufferedLogger(org.slf4j.Logger wrappedLogger) {
this.wrappedLogger = wrappedLogger;
}
@@ -34,7 +36,9 @@ public class BufferedLogger implements Logger {
@Override
public void log(String message) {
wrappedLogger.info(message);
- buffer.append(message).append("\n");
+ if (buffer.length() < MAX_BUFFER_SIZE) {
+ buffer.append(message).append("\n");
+ }
}
public String getBufferedLog() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
new file mode 100644
index 0000000..8399b44
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.common;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.common.util.BufferedLogger;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.slf4j.Logger;
+
+import com.google.common.collect.Maps;
+
+/**
+ * A logger which parses certain patterns from log
+ */
+public class PatternedLogger extends BufferedLogger {
+ private final Map<String, String> info = Maps.newHashMap();
+
+ private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager");
+ private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)");
+ private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)");
+ private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
+ private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)");
+ private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write");
+
+ // hive
+ private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
+ private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+
+ // spark
+ private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
+ private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
+
+
+ public PatternedLogger(Logger wrappedLogger) {
+ super(wrappedLogger);
+ }
+
+ @Override
+ public void log(String message) {
+ super.log(message);
+ Matcher matcher = PATTERN_APP_ID.matcher(message);
+ if (matcher.find()) {
+ String appId = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_ID, appId);
+ }
+
+ matcher = PATTERN_APP_URL.matcher(message);
+ if (matcher.find()) {
+ String appTrackingUrl = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_URL, appTrackingUrl);
+ }
+
+ matcher = PATTERN_JOB_ID.matcher(message);
+ if (matcher.find()) {
+ String mrJobID = matcher.group(1);
+ info.put(ExecutableConstants.MR_JOB_ID, mrJobID);
+ }
+
+ matcher = PATTERN_HDFS_BYTES_WRITTEN.matcher(message);
+ if (matcher.find()) {
+ String hdfsWritten = matcher.group(1);
+ info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
+ }
+
+ matcher = PATTERN_SOURCE_RECORDS_COUNT.matcher(message);
+ if (matcher.find()) {
+ String sourceCount = matcher.group(1);
+ info.put(ExecutableConstants.SOURCE_RECORDS_COUNT, sourceCount);
+ }
+
+ matcher = PATTERN_SOURCE_RECORDS_SIZE.matcher(message);
+ if (matcher.find()) {
+ String sourceSize = matcher.group(1);
+ info.put(ExecutableConstants.SOURCE_RECORDS_SIZE, sourceSize);
+ }
+
+ // hive
+ matcher = PATTERN_HIVE_APP_ID_URL.matcher(message);
+ if (matcher.find()) {
+ String jobId = matcher.group(1);
+ String trackingUrl = matcher.group(2);
+ info.put(ExecutableConstants.MR_JOB_ID, jobId);
+ info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+ }
+
+ matcher = PATTERN_HIVE_BYTES_WRITTEN.matcher(message);
+ if (matcher.find()) {
+ // String hdfsRead = matcher.group(1);
+ String hdfsWritten = matcher.group(2);
+ info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
+ }
+
+ // spark
+ matcher = PATTERN_SPARK_APP_ID.matcher(message);
+ if (matcher.find()) {
+ String app_id = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_ID, app_id);
+ }
+
+ matcher = PATTERN_SPARK_APP_URL.matcher(message);
+ if (matcher.find()) {
+ String trackingUrl = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+ }
+ }
+
+ public Map<String, String> getInfo() {
+ return info;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 5e0d0db..9f431b0 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -19,21 +19,13 @@
package org.apache.kylin.job.common;
import java.io.IOException;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.kylin.common.util.Logger;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
-
/**
*/
public class ShellExecutable extends AbstractExecutable {
@@ -50,9 +42,9 @@ public class ShellExecutable extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
try {
logger.info("executing:" + getCmd());
- final ShellExecutableLogger logger = new ShellExecutableLogger();
- final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), logger);
- getManager().addJobInfo(getId(), logger.getInfo());
+ final PatternedLogger patternedLogger = new PatternedLogger(logger);
+ final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), patternedLogger);
+ getManager().addJobInfo(getId(), patternedLogger.getInfo());
return new ExecuteResult(result.getFirst() == 0 ? ExecuteResult.State.SUCCEED : ExecuteResult.State.FAILED, result.getSecond());
} catch (IOException e) {
logger.error("job:" + getId() + " execute finished with exception", e);
@@ -68,96 +60,4 @@ public class ShellExecutable extends AbstractExecutable {
return getParam(CMD);
}
- private static class ShellExecutableLogger implements Logger {
-
- private final Map<String, String> info = Maps.newHashMap();
-
- private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager");
- private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)");
- private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)");
- private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
- private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)");
- private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write");
-
- // hive
- private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
- private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
-
- // spark
- private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
- private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
-
- @Override
- public void log(String message) {
- Matcher matcher = PATTERN_APP_ID.matcher(message);
- if (matcher.find()) {
- String appId = matcher.group(1);
- info.put(ExecutableConstants.YARN_APP_ID, appId);
- }
-
- matcher = PATTERN_APP_URL.matcher(message);
- if (matcher.find()) {
- String appTrackingUrl = matcher.group(1);
- info.put(ExecutableConstants.YARN_APP_URL, appTrackingUrl);
- }
-
- matcher = PATTERN_JOB_ID.matcher(message);
- if (matcher.find()) {
- String mrJobID = matcher.group(1);
- info.put(ExecutableConstants.MR_JOB_ID, mrJobID);
- }
-
- matcher = PATTERN_HDFS_BYTES_WRITTEN.matcher(message);
- if (matcher.find()) {
- String hdfsWritten = matcher.group(1);
- info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
- }
-
- matcher = PATTERN_SOURCE_RECORDS_COUNT.matcher(message);
- if (matcher.find()) {
- String sourceCount = matcher.group(1);
- info.put(ExecutableConstants.SOURCE_RECORDS_COUNT, sourceCount);
- }
-
- matcher = PATTERN_SOURCE_RECORDS_SIZE.matcher(message);
- if (matcher.find()) {
- String sourceSize = matcher.group(1);
- info.put(ExecutableConstants.SOURCE_RECORDS_SIZE, sourceSize);
- }
-
- // hive
- matcher = PATTERN_HIVE_APP_ID_URL.matcher(message);
- if (matcher.find()) {
- String jobId = matcher.group(1);
- String trackingUrl = matcher.group(2);
- info.put(ExecutableConstants.MR_JOB_ID, jobId);
- info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
- }
-
- matcher = PATTERN_HIVE_BYTES_WRITTEN.matcher(message);
- if (matcher.find()) {
- // String hdfsRead = matcher.group(1);
- String hdfsWritten = matcher.group(2);
- info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
- }
-
- // spark
- matcher = PATTERN_SPARK_APP_ID.matcher(message);
- if (matcher.find()) {
- String app_id = matcher.group(1);
- info.put(ExecutableConstants.YARN_APP_ID, app_id);
- }
-
- matcher = PATTERN_SPARK_APP_URL.matcher(message);
- if (matcher.find()) {
- String trackingUrl = matcher.group(1);
- info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
- }
- }
-
- Map<String, String> getInfo() {
- return info;
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 9cb60f8..b54f401 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -224,7 +224,7 @@ public class CubeStatsReader {
double cuboidSizeRatio = kylinConf.getJobCuboidSizeRatio();
double cuboidSizeMemHungryRatio = kylinConf.getJobCuboidSizeCountDistinctRatio();
double ret = (1.0 * normalSpace * rowCount * cuboidSizeRatio + 1.0 * countDistinctSpace * rowCount * cuboidSizeMemHungryRatio) / (1024L * 1024L);
- logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + (normalSpace + countDistinctSpace) + " bytes." + " Total size is " + ret + "M.");
+ logger.debug("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + (normalSpace + countDistinctSpace) + " bytes." + " Total size is " + ret + "M.");
return ret;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 5ad21cf..1ed2235 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.Logger;
+import org.apache.kylin.job.common.PatternedLogger;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -118,22 +118,17 @@ public class SparkExecutable extends AbstractExecutable {
stringBuilder.append("--files %s --jars %s %s %s");
try {
String cmd = String.format(stringBuilder.toString(), hadoopConf, KylinConfig.getSparkHome(), hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
- logger.info("cmd:" + cmd);
- final StringBuilder output = new StringBuilder();
+ logger.info("cmd: " + cmd);
CliCommandExecutor exec = new CliCommandExecutor();
- exec.execute(cmd, new Logger() {
- @Override
- public void log(String message) {
- output.append(message);
- output.append("\n");
- logger.info(message);
- }
- });
- return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
+ PatternedLogger patternedLogger = new PatternedLogger(logger);
+ exec.execute(cmd, patternedLogger);
+ getManager().addJobInfo(getId(), patternedLogger.getInfo());
+ return new ExecuteResult(ExecuteResult.State.SUCCEED, patternedLogger.getBufferedLog());
} catch (Exception e) {
logger.error("error run spark job:", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index b197f0e..8b241d2 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -20,12 +20,12 @@ package org.apache.kylin.source.hive;
import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.common.PatternedLogger;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
public class CreateFlatHiveTableStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(CreateFlatHiveTableStep.class);
- private final BufferedLogger stepLogger = new BufferedLogger(logger);
+ private final PatternedLogger stepLogger = new PatternedLogger(logger);
protected void createFlatHiveTable(KylinConfig config) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
@@ -50,6 +50,7 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
stepLogger.log(cmd);
Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger);
+ getManager().addJobInfo(getId(), stepLogger.getInfo());
if (response.getFirst() != 0) {
throw new RuntimeException("Failed to create flat hive table, error code " + response.getFirst());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/47b7c800/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 39a0273..bbf3c60 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.Pair;
@@ -40,6 +39,7 @@ import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JoinedFlatTable;
+import org.apache.kylin.job.common.PatternedLogger;
import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -246,7 +246,7 @@ public class HiveMRInput implements IMRInput {
}
public static class RedistributeFlatHiveTableStep extends AbstractExecutable {
- private final BufferedLogger stepLogger = new BufferedLogger(logger);
+ private final PatternedLogger stepLogger = new PatternedLogger(logger);
private long computeRowCount(String database, String table) throws Exception {
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
@@ -265,6 +265,8 @@ public class HiveMRInput implements IMRInput {
stepLogger.log(cmd);
Pair<Integer, String> response = config.getCliCommandExecutor().execute(cmd, stepLogger);
+ getManager().addJobInfo(getId(), stepLogger.getInfo());
+
if (response.getFirst() != 0) {
throw new RuntimeException("Failed to redistribute flat hive table");
}
[43/48] kylin git commit: bug in BuildCubeWithStream in last commit
Posted by li...@apache.org.
bug in BuildCubeWithStream in last commit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0eca241
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0eca241
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0eca241
Branch: refs/heads/master-hbase0.98
Commit: f0eca24140b2a48e43b287bafe08f37e138e2df0
Parents: 530549c
Author: shaofengshi <sh...@apache.org>
Authored: Sun Mar 12 12:54:18 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 12 12:54:18 2017 +0800
----------------------------------------------------------------------
.../kylin/provision/BuildCubeWithStream.java | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0eca241/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 2880bf4..c5341d6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -198,15 +198,16 @@ public class BuildCubeWithStream {
if (i == (BUILD_ROUND - 1)) {
// stop generating message to kafka
generateData = false;
+ int waittime = 0;
+ while (generateDataDone == false && waittime < 100) {
+ Thread.sleep(1000);
+ waittime++;
+ }
+ if (generateDataDone == false) {
+ throw new IllegalStateException("Timeout when wait all messages be sent to Kafka"); // ensure all messages have been flushed.
+ }
}
- int waittime = 0;
- while (generateDataDone == false && waittime < 100) {
- Thread.sleep(1000);
- waittime++;
- }
- if (generateDataDone == false) {
- throw new IllegalStateException("Timeout when wait all messages be sent to Kafka"); // ensure all messages have been flushed.
- }
+
FutureTask futureTask = new FutureTask(new Callable<ExecutableState>() {
@Override
public ExecutableState call() {
[06/48] kylin git commit: KYLIN-2471 queries with parenthesized
sub-clause in JOIN will fail
Posted by li...@apache.org.
KYLIN-2471 queries with parenthesized sub-clause in JOIN will fail
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29b3f6d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29b3f6d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29b3f6d4
Branch: refs/heads/master-hbase0.98
Commit: 29b3f6d422f0d4c86678b574864e8823892e3924
Parents: d6fc052
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Feb 25 16:36:19 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Feb 25 16:36:45 2017 +0800
----------------------------------------------------------------------
.../rest/util/CognosParenthesesEscape.java | 83 ++++++++++++++++++++
.../rest/util/CognosParentesesEscapeTest.java | 61 ++++++++++++++
.../src/test/resources/query/cognos/query01.sql | 28 +++++++
.../resources/query/cognos/query01.sql.expected | 28 +++++++
4 files changed, 200 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3f6d4/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
new file mode 100644
index 0000000..33f4b63
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer;
+
+/**
+ * from (a join b on a.x = b.y) join c
+ *
+ * similar in https://issues.apache.org/jira/browse/CALCITE-35
+ *
+ * we'll find such pattern and remove the parentheses
+ */
+public class CognosParenthesesEscape implements IQueryTransformer {
+
+ private static final String S0 = "\\s*";
+ private static final String S1 = "\\s";
+ private static final String SM = "\\s+";
+ private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+";
+ private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?";
+ private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server
+ private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME;
+ private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // (
+ TABLE_NAME_WITH_OPTIONAL_ALIAS + // a
+ JOIN + SM + // join
+ TABLE_NAME_WITH_OPTIONAL_ALIAS + //b
+ SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2]
+ S0 + "\\)";// )
+ private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE);
+
+ private static int identifierNum = 0;
+
+ @Override
+ public String transform(String sql) {
+ Matcher m;
+ List<Pair<String, String>> matches = new LinkedList<>();
+ while (true) {
+ m = PARENTTHESES_PATTERN.matcher(sql);
+ if (!m.find())
+ break;
+
+ String oneParentheses = m.group(0);
+ String identifier = generateRandomName();
+ matches.add(new Pair<String, String>(identifier, oneParentheses.substring(1, oneParentheses.length() - 1)));
+ sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end());
+ }
+
+ for (int i = matches.size() - 1; i >= 0; i--) {
+ sql = sql.replaceAll(matches.get(i).getKey(), matches.get(i).getValue());
+ }
+
+ return sql;
+ }
+
+ private String generateRandomName() {
+ UUID uuid = UUID.randomUUID();
+ return uuid.toString().replace("-", "_") + "_" + (identifierNum++);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3f6d4/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java
new file mode 100644
index 0000000..76df219
--- /dev/null
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.rest.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Collection;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CognosParentesesEscapeTest {
+
+ @Test
+ public void basicTest() {
+ CognosParenthesesEscape escape = new CognosParenthesesEscape();
+ String data = "((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
+ String expected = "a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
+ String transformed = escape.transform(data);
+ Assert.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void advancedTest() throws IOException {
+ CognosParenthesesEscape escape = new CognosParenthesesEscape();
+ String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"), Charset.defaultCharset());
+ String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"), Charset.defaultCharset());
+ String transformed = escape.transform(query);
+ //System.out.println(transformed);
+ Assert.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void proguardTest() throws IOException {
+ CognosParenthesesEscape escape = new CognosParenthesesEscape();
+ Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" }, true);
+ for (File f : files) {
+ System.out.println("checking " + f.getAbsolutePath());
+ String query = FileUtils.readFileToString(f, Charset.defaultCharset());
+ String transformed = escape.transform(query);
+ Assert.assertEquals(query, transformed);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3f6d4/server-base/src/test/resources/query/cognos/query01.sql
----------------------------------------------------------------------
diff --git a/server-base/src/test/resources/query/cognos/query01.sql b/server-base/src/test/resources/query/cognos/query01.sql
new file mode 100644
index 0000000..a76b4e1
--- /dev/null
+++ b/server-base/src/test/resources/query/cognos/query01.sql
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT "TABLE1"."DIM1_1" "DIM1_1"
+ ,"TABLE2"."DIM2_1" "DIM2_1"
+ ,SUM("FACT"."M1") "M1"
+ ,SUM("FACT"."M2") "M2"
+ FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
+ "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1")
+ LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
+ ON "FACT"."FK_2" = "TABLE2"."PK_2"
+ GROUP BY "TABLE2"."DIM2_1"
+ ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/29b3f6d4/server-base/src/test/resources/query/cognos/query01.sql.expected
----------------------------------------------------------------------
diff --git a/server-base/src/test/resources/query/cognos/query01.sql.expected b/server-base/src/test/resources/query/cognos/query01.sql.expected
new file mode 100644
index 0000000..c8005fc
--- /dev/null
+++ b/server-base/src/test/resources/query/cognos/query01.sql.expected
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT "TABLE1"."DIM1_1" "DIM1_1"
+ ,"TABLE2"."DIM2_1" "DIM2_1"
+ ,SUM("FACT"."M1") "M1"
+ ,SUM("FACT"."M2") "M2"
+ FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
+ "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1"
+ LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
+ ON "FACT"."FK_2" = "TABLE2"."PK_2"
+ GROUP BY "TABLE2"."DIM2_1"
+ ,"TABLE1"."DIM1_1";