You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/08 06:03:38 UTC
[1/8] kylin git commit: KYLIN-1875 Kylin support SnowFlake schema
(alias name rule) [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2428 d995caad5 -> 6d5ed7ec1 (forced update)
KYLIN-1875 Kylin support SnowFlake schema (alias name rule)
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8581f1d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8581f1d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8581f1d2
Branch: refs/heads/KYLIN-2428
Commit: 8581f1d203cee9fe3d834d6beb9f6bf7717f921a
Parents: 7a6dd1c
Author: luguosheng <55...@qq.com>
Authored: Tue Feb 7 16:24:23 2017 +0800
Committer: zhongjian <ji...@163.com>
Committed: Tue Feb 7 17:08:24 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/modelDesigner/data_model.html | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8581f1d2/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 4d72266..167c088 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -157,8 +157,8 @@
<label class="col-sm-3 control-label font-color-default"><b>Alias</b></label>
<div class="col-sm-6">
<input type="text" class="form-control " name="joinTable_alias" placeholder="Input Table Alias" ng-required="true"
- ng-model="newLookup.alias" ng-pattern="/^\w+$/">
- <small class="help-block red" ng-show="!lookup_form.joinTable_alias.$error.required&&lookup_form.joinTable_alias.$invalid && (lookup_form.joinTable_alias.$dirty||lookup_form.$submitted)"><i class="fa fa-exclamation-triangle"></i> Alias is invalid.</small>
+ ng-model="newLookup.alias" ng-pattern="/^[A-Z_\d]+$/">
+ <small class="help-block red" ng-show="!lookup_form.joinTable_alias.$error.required&&lookup_form.joinTable_alias.$invalid && (lookup_form.joinTable_alias.$dirty||lookup_form.$submitted)"><i class="fa fa-exclamation-triangle"></i> Table alias is invalid(A Combination of numbers, uppercase letters or underscores).</small>
</div>
</div>
</div>
[3/8] kylin git commit: KYLIN-2433 Fix NPE in MergeCuboidMapper
Posted by bi...@apache.org.
KYLIN-2433 Fix NPE in MergeCuboidMapper
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b32cc954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b32cc954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b32cc954
Branch: refs/heads/KYLIN-2428
Commit: b32cc9545a76657570c4a3353469ded43892c772
Parents: 6d11dd1
Author: kangkaisen <ka...@163.com>
Authored: Fri Feb 3 14:00:50 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Tue Feb 7 17:20:30 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/steps/MergeCuboidMapper.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b32cc954/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 047e2b1..acf1403 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -129,6 +129,11 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
List<TblColRef> columns = measureType.getColumnsNeedDictionary(measureDesc.getFunction());
boolean needReEncode = false;
for (TblColRef col : columns) {
+ //handle the column that all records is null
+ if (sourceCubeSegment.getDictionary(col) == null) {
+ continue;
+ }
+
if (!sourceCubeSegment.getDictionary(col).equals(mergedCubeSegment.getDictionary(col))) {
oldDicts.put(col, sourceCubeSegment.getDictionary(col));
newDicts.put(col, mergedCubeSegment.getDictionary(col));
@@ -249,6 +254,11 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
}
private Boolean checkNeedMerging(TblColRef col) throws IOException {
+ //handle the column that all records is null
+ if (sourceCubeSegment.getDictionary(col) == null) {
+ return false;
+ }
+
Boolean ret = dimensionsNeedDict.get(col);
if (ret != null)
return ret;
[6/8] kylin git commit: KYLIN-2430 Get exitCode form FsShell.run in
BulkLoadJob
Posted by bi...@apache.org.
KYLIN-2430 Get exitCode form FsShell.run in BulkLoadJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b860adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b860adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b860adb
Branch: refs/heads/KYLIN-2428
Commit: 7b860adb0a23b53d6e2e40337f4317a3a2b067ba
Parents: eee9ecb
Author: kangkaisen <ka...@163.com>
Authored: Tue Feb 7 17:35:48 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Tue Feb 7 17:40:39 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/steps/BulkLoadJob.java | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b860adb/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 1c05767..30616c5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -55,11 +55,18 @@ public class BulkLoadJob extends AbstractHadoopJob {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
FsShell shell = new FsShell(conf);
- try {
- shell.run(new String[] { "-chmod", "-R", "777", input });
- } catch (Exception e) {
- logger.error("Couldn't change the file permissions ", e);
- throw new IOException(e);
+
+ int exitCode = -1;
+ int retryCount = 10;
+ while (exitCode != 0 && retryCount >= 1) {
+ exitCode = shell.run(new String[] { "-chmod", "-R", "777", input });
+ retryCount--;
+ Thread.sleep(5000);
+ }
+
+ if (exitCode != 0) {
+ logger.error("Failed to change the file permissions: " + input);
+ throw new IOException("Failed to change the file permissions: " + input);
}
String[] newArgs = new String[2];
[2/8] kylin git commit: KYLIN-2432 Couldn't select partition column
in some old browser
Posted by bi...@apache.org.
KYLIN-2432 Couldn't select partition column in some old browser
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d11dd1d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d11dd1d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d11dd1d
Branch: refs/heads/KYLIN-2428
Commit: 6d11dd1d27a74fe50cd4fa58d2acf26f728f82e7
Parents: 8581f1d
Author: luguosheng <55...@qq.com>
Authored: Tue Feb 7 16:54:05 2017 +0800
Committer: zhongjian <ji...@163.com>
Committed: Tue Feb 7 17:08:45 2017 +0800
----------------------------------------------------------------------
webapp/app/js/config.js | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d11dd1d/webapp/app/js/config.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/config.js b/webapp/app/js/config.js
index 5f72eb2..b9ae6a5 100644
--- a/webapp/app/js/config.js
+++ b/webapp/app/js/config.js
@@ -35,7 +35,17 @@ var Config = {
},
contact_mail: ''
};
-
+//resolve startsWith and endsWidth not work in low version chrome
+if (typeof String.prototype.startsWith != 'function') {
+ String.prototype.startsWith = function (prefix){
+ return this.slice(0, prefix.length) === prefix;
+ };
+}
+if (typeof String.prototype.endsWith != 'function') {
+ String.prototype.endsWith = function(suffix) {
+ return this.indexOf(suffix, this.length - suffix.length) !== -1;
+ };
+}
// Angular module to load routes.
KylinApp.config(function ($routeProvider, $httpProvider, $locationProvider, $logProvider) {
//resolve http always use cache data in IE11,IE10
[7/8] kylin git commit: KYLIN-2388 Hot load kylin config from web
Posted by bi...@apache.org.
KYLIN-2388 Hot load kylin config from web
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39afa519
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39afa519
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39afa519
Branch: refs/heads/KYLIN-2428
Commit: 39afa5197e197d5143843a3bb62c528749b03418
Parents: 7b860ad
Author: kangkaisen <ka...@live.com>
Authored: Thu Jan 12 14:35:59 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Tue Feb 7 19:57:29 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 6 +-
.../apache/kylin/common/KylinConfigTest.java | 28 ++++-----
.../util/HotLoadKylinPropertiesTestCase.java | 64 ++++++++++++++++++++
.../kylin/cube/CubeSpecificConfigTest.java | 37 ++++++-----
.../apache/kylin/job/JobEngineConfigTest.java | 47 ++++++++++++++
.../kylin/rest/controller/CacheController.java | 9 ++-
webapp/app/js/controllers/admin.js | 26 ++++++++
webapp/app/js/services/cache.js | 3 +-
webapp/app/partials/admin/admin.html | 3 +
9 files changed, 191 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0f40654..c7a18c6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -201,7 +201,7 @@ public class KylinConfig extends KylinConfigBase {
return kylinHome + File.separator + "conf";
}
- static File getKylinPropertiesFile() {
+ public static File getKylinPropertiesFile() {
String kylinConfHome = System.getProperty(KYLIN_CONF);
if (!StringUtils.isEmpty(kylinConfHome)) {
logger.info("Use KYLIN_CONF=" + kylinConfHome);
@@ -385,4 +385,8 @@ public class KylinConfig extends KylinConfigBase {
// }
// logger.info(buf.toString());
}
+
+ public synchronized void hotLoadKylinProperties() {
+ reloadKylinConfig(getKylinProperties());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index a426fc6..4d5f130 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -26,24 +26,12 @@ import static org.junit.Assert.assertTrue;
import java.util.Map;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.junit.Test;
import com.google.common.collect.Maps;
-public class KylinConfigTest extends LocalFileMetadataTestCase {
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
+public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
@Test
public void testMRConfigOverride() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -78,9 +66,19 @@ public class KylinConfigTest extends LocalFileMetadataTestCase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
Map<String, String> override = Maps.newHashMap();
KylinConfig configExt = KylinConfigExt.createInstance(config, override);
-
assertTrue(config.properties == configExt.properties);
config.setProperty("1234", "1234");
assertEquals("1234", configExt.getOptional("1234"));
}
+
+ @Test
+ public void testPropertiesHotLoad() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ assertEquals("whoami@kylin.apache.org", config.getKylinOwner());
+
+ updateProperty("kylin.storage.hbase.owner-tag", "kylin@kylin.apache.org");
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+ assertEquals("kylin@kylin.apache.org", config.getKylinOwner());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
new file mode 100644
index 0000000..9f5b278
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.apache.kylin.common.KylinConfig;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.Properties;
+
+/**
+ * @author kangkaisen
+ */
+
+public class HotLoadKylinPropertiesTestCase extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ protected void updateProperty(String key, String value) {
+ File propFile = KylinConfig.getKylinPropertiesFile();
+ Properties conf = new Properties();
+
+ //load
+ try (FileInputStream is = new FileInputStream(propFile)) {
+ conf.load(is);
+ conf.setProperty(key, value);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ }
+
+ //store
+ try (FileOutputStream out = new FileOutputStream(propFile)) {
+ conf.store(out, null);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
index c61f07f..17c02cc 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
@@ -21,24 +21,13 @@ package org.apache.kylin.cube;
import static org.junit.Assert.assertEquals;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.apache.kylin.cube.model.CubeDesc;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
+import java.io.IOException;
+public class CubeSpecificConfigTest extends HotLoadKylinPropertiesTestCase {
@Test
public void test() {
KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
@@ -57,4 +46,24 @@ public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
assertEquals("snappy", base.getHbaseDefaultCompressionCodec());
assertEquals("lz4", override.getHbaseDefaultCompressionCodec());
}
+
+ @Test
+ public void testPropertiesHotLoad() throws IOException {
+ KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+ KylinConfig oldCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig();
+ assertEquals(10, oldCubeDescConfig.getMaxConcurrentJobLimit());
+
+ //hot load Properties
+ updateProperty("kylin.job.max-concurrent-jobs", "20");
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+ CubeDescManager.getInstance(baseConfig).reloadCubeDescLocal("ssb");
+
+ //test cubeDescConfig
+ KylinConfig newCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig();
+ assertEquals(20, newCubeDescConfig.getMaxConcurrentJobLimit());
+
+ //test cubeConfig
+ KylinConfig newCubeConfig = CubeManager.getInstance(baseConfig).getCube("ssb").getConfig();
+ assertEquals(20, newCubeConfig.getMaxConcurrentJobLimit());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java
new file mode 100644
index 0000000..77914ef
--- /dev/null
+++ b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author kangkaisen
+ */
+
+public class JobEngineConfigTest extends HotLoadKylinPropertiesTestCase {
+
+ @Test
+ public void testPropertiesHotLoad() throws IOException {
+ KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(baseConfig);
+ assertEquals(10, jobEngineConfig.getMaxConcurrentJobLimit());
+
+ updateProperty("kylin.job.max-concurrent-jobs", "20");
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+ assertEquals(20, jobEngineConfig.getMaxConcurrentJobLimit());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
index 254aabf..8d5f00e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.rest.service.CacheService;
import org.slf4j.Logger;
@@ -39,7 +40,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
@Controller
@RequestMapping(value = "/cache")
public class CacheController extends BasicController {
-
+
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(CacheController.class);
@@ -64,6 +65,12 @@ public class CacheController extends BasicController {
cacheService.notifyMetadataChange(entity, Broadcaster.Event.getEvent(event), cacheKey);
}
+ @RequestMapping(value = "/announce/config", method = { RequestMethod.POST })
+ public void hotLoadKylinConfig() throws IOException {
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+ cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL);
+ }
+
public void setCacheService(CacheService cacheService) {
this.cacheService = cacheService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/js/controllers/admin.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/admin.js b/webapp/app/js/controllers/admin.js
index 0d36e0d..783ab17 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -58,6 +58,32 @@ KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService, T
});
}
+ $scope.reloadConfig = function () {
+ SweetAlert.swal({
+ title: '',
+ text: 'Are you sure to reload config',
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function (isConfirm) {
+ if (isConfirm) {
+ CacheService.reloadConfig({}, function () {
+ SweetAlert.swal('Success!', 'config reload successfully', 'success');
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
+ }
+ });
+ }
+
$scope.reloadMeta = function () {
SweetAlert.swal({
title: '',
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/js/services/cache.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/cache.js b/webapp/app/js/services/cache.js
index bcec603..38dc0b8 100644
--- a/webapp/app/js/services/cache.js
+++ b/webapp/app/js/services/cache.js
@@ -18,6 +18,7 @@
KylinApp.factory('CacheService', ['$resource', function ($resource, config) {
return $resource(Config.service.url + 'cache/announce/:type/:name/:action', {}, {
- clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false}
+ clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false},
+ reloadConfig: {method: 'POST', params: {type: 'config'}, isArray: false}
});
}]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/partials/admin/admin.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/admin/admin.html b/webapp/app/partials/admin/admin.html
index f2be3d9..b4bca8d 100644
--- a/webapp/app/partials/admin/admin.html
+++ b/webapp/app/partials/admin/admin.html
@@ -58,6 +58,9 @@
<a class="btn btn-primary btn-lg btn-block" tooltip="Update Server Config" class="btn btn-primary btn-lg" ng-click="toSetConfig()">Set Config</a>
</div>
<div style="padding-top: 10px;width: 260px;">
+ <a class="btn btn-primary btn-lg btn-block" tooltip="Reload Server Config" class="btn btn-primary btn-lg" ng-click="reloadConfig()">Reload Config</a>
+ </div>
+ <div style="padding-top: 10px;width: 260px;">
<a ng-click="downloadBadQueryFiles();" tooltip="Download Diagnosis Info For Current Project" class="btn btn-primary btn-lg btn-block"><i class="fa fa-ambulance"></i> Diagnosis</a>
</div>
[8/8] kylin git commit: KYLIN-2428 cleanup duplicated shaded libraries
Posted by bi...@apache.org.
KYLIN-2428 cleanup duplicated shaded libraries
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d5ed7ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d5ed7ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d5ed7ec
Branch: refs/heads/KYLIN-2428
Commit: 6d5ed7ec16308ec2a2c910710449d0b79e41da84
Parents: 39afa51
Author: Billy Liu <bi...@apache.org>
Authored: Wed Feb 8 14:03:23 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Feb 8 14:03:23 2017 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 1 +
build/bin/kylin.sh | 4 +-
core-common/pom.xml | 22 +-
core-cube/pom.xml | 18 +-
.../apache/kylin/gridtable/GTInvertedIndex.java | 223 -------------------
.../gridtable/GTInvertedIndexOfColumn.java | 133 -----------
.../gridtable/SimpleInvertedIndexTest.java | 196 ----------------
core-job/pom.xml | 9 +
core-metadata/pom.xml | 23 ++
core-storage/pom.xml | 20 ++
engine-spark/pom.xml | 1 +
jdbc/pom.xml | 14 +-
pom.xml | 214 ++++++++++--------
query/pom.xml | 6 +
server/pom.xml | 4 +
source-kafka/pom.xml | 16 +-
storage-hbase/pom.xml | 4 -
tool/pom.xml | 33 +++
18 files changed, 251 insertions(+), 690 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop -->
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7813b79..abaca62 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,8 +38,8 @@ function retrieveDependency() {
source ${dir}/setenv.sh
fi
- export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
- export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+ export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${KYLIN_HOME}/tool/*
if [ -n "$KAFKA_HOME" ]
then
source ${dir}/find-kafka-dependency.sh
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
<dependencies>
<!-- Basic Utilities -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
</dependency>
@@ -63,9 +47,11 @@
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
</dependency>
+
+ <!-- Provided -->
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
+
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
deleted file mode 100644
index a2c713a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * A thread-safe inverted index of row blocks in memory.
- *
- * Note function not() must return all blocks, because index only know what block contains a value,
- * but not sure what block does not contain a value.
- *
- * @author yangli9
- */
-public class GTInvertedIndex {
-
- private final GTInfo info;
- private final ImmutableBitSet colPreferIndex;
- private final ImmutableBitSet colBlocks;
- private final GTInvertedIndexOfColumn[] index; // for each column
-
- private volatile int nIndexedBlocks;
-
- public GTInvertedIndex(GTInfo info) {
- this.info = info;
- this.colPreferIndex = info.colPreferIndex;
- this.colBlocks = info.selectColumnBlocks(colPreferIndex);
-
- index = new GTInvertedIndexOfColumn[info.getColumnCount()];
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator());
- }
- }
-
- public void add(GTRowBlock block) {
-
- @SuppressWarnings("unchecked")
- Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- distinctValues[c] = new HashSet<ByteArray>();
- }
-
- GTRowBlock.Reader reader = block.getReader(colBlocks);
- GTRecord record = new GTRecord(info);
- while (reader.hasNext()) {
- reader.fetchNext(record);
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- distinctValues[c].add(record.get(c));
- }
- }
-
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- index[c].add(distinctValues[c], block.getSequenceId());
- }
-
- nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
- }
-
- public ConciseSet filter(TupleFilter filter) {
- return filter(filter, nIndexedBlocks);
- }
-
- public ConciseSet filter(TupleFilter filter, int totalBlocks) {
- // number of indexed blocks may increase as we do evaluation
- int indexedBlocks = nIndexedBlocks;
-
- Evaluator evaluator = new Evaluator(indexedBlocks);
- ConciseSet r = evaluator.evaluate(filter);
-
- // add blocks that have not been indexed
- for (int i = indexedBlocks; i < totalBlocks; i++) {
- r.add(i);
- }
-
- return r;
- }
-
- private class Evaluator {
- private int indexedBlocks;
-
- Evaluator(int indexedBlocks) {
- this.indexedBlocks = indexedBlocks;
- }
-
- public ConciseSet evaluate(TupleFilter filter) {
- if (filter == null) {
- return all();
- }
-
- if (filter instanceof LogicalTupleFilter)
- return evalLogical((LogicalTupleFilter) filter);
-
- if (filter instanceof CompareTupleFilter)
- return evalCompare((CompareTupleFilter) filter);
-
- // unable to evaluate
- return all();
- }
-
- @SuppressWarnings("unchecked")
- private ConciseSet evalCompare(CompareTupleFilter filter) {
- int col = col(filter);
- if (index[col] == null)
- return all();
-
- switch (filter.getOperator()) {
- case ISNULL:
- return index[col].getNull();
- case ISNOTNULL:
- return all();
- case EQ:
- return index[col].getEquals((ByteArray) filter.getFirstValue());
- case NEQ:
- return all();
- case IN:
- return index[col].getIn((Iterable<ByteArray>) filter.getValues());
- case NOTIN:
- return all();
- case LT:
- return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false);
- case LTE:
- return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true);
- case GT:
- return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false);
- case GTE:
- return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false);
- default:
- throw new IllegalStateException("Unsupported operator " + filter.getOperator());
- }
- }
-
- private ConciseSet evalLogical(LogicalTupleFilter filter) {
- List<? extends TupleFilter> children = filter.getChildren();
-
- switch (filter.getOperator()) {
- case AND:
- return evalLogicalAnd(children);
- case OR:
- return evalLogicalOr(children);
- case NOT:
- return evalLogicalNot(children);
- default:
- throw new IllegalStateException("Unsupported operator " + filter.getOperator());
- }
- }
-
- private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) {
- ConciseSet set = all();
-
- for (TupleFilter c : children) {
- ConciseSet t = evaluate(c);
- if (t == null)
- continue; // because it's AND
-
- set.retainAll(t);
- }
- return set;
- }
-
- private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) {
- ConciseSet set = new ConciseSet();
-
- for (TupleFilter c : children) {
- ConciseSet t = evaluate(c);
- if (t == null)
- return null; // because it's OR
-
- set.addAll(t);
- }
- return set;
- }
-
- private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) {
- return all();
- }
-
- private ConciseSet all() {
- return not(new ConciseSet());
- }
-
- private ConciseSet not(ConciseSet set) {
- set.add(indexedBlocks);
- set.complement();
- return set;
- }
-
- private int col(CompareTupleFilter filter) {
- return filter.getColumn().getColumnDesc().getZeroBasedIndex();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
deleted file mode 100644
index bfacc0f..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.NavigableMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import com.google.common.collect.Maps;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class GTInvertedIndexOfColumn {
-
- final private IGTComparator comparator;
- final private ReentrantReadWriteLock rwLock;
-
- private int nBlocks;
- private NavigableMap<ByteArray, ConciseSet> rangeIndex;
- private ConciseSet nullIndex;
-
- public GTInvertedIndexOfColumn(IGTComparator comparator) {
- this.comparator = comparator;
- this.rwLock = new ReentrantReadWriteLock();
- this.rangeIndex = Maps.newTreeMap(comparator);
- this.nullIndex = new ConciseSet();
- }
-
- public void add(Iterable<ByteArray> codes, int blockId) {
- rwLock.writeLock().lock();
- try {
- for (ByteArray code : codes) {
- if (comparator.isNull(code)) {
- nullIndex.add(blockId);
- continue;
- }
- ConciseSet set = rangeIndex.get(code);
- if (set == null) {
- set = new ConciseSet();
- rangeIndex.put(code.copy(), set);
- }
- set.add(blockId);
- }
-
- if (blockId >= nBlocks) {
- nBlocks = blockId + 1;
- }
-
- } finally {
- rwLock.writeLock().unlock();
- }
- }
-
- public ConciseSet getNull() {
- rwLock.readLock().lock();
- try {
- return nullIndex.clone();
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getEquals(ByteArray code) {
- rwLock.readLock().lock();
- try {
- ConciseSet set = rangeIndex.get(code);
- if (set == null)
- return new ConciseSet();
- else
- return set.clone();
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getIn(Iterable<ByteArray> codes) {
- rwLock.readLock().lock();
- try {
- ConciseSet r = new ConciseSet();
- for (ByteArray code : codes) {
- ConciseSet set = rangeIndex.get(code);
- if (set != null)
- r.addAll(set);
- }
- return r;
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) {
- rwLock.readLock().lock();
- try {
- ConciseSet r = new ConciseSet();
- if (from == null && to == null) {
- r.add(nBlocks);
- r.complement();
- return r;
- }
- NavigableMap<ByteArray, ConciseSet> subMap;
- if (from == null) {
- subMap = rangeIndex.headMap(to, toInclusive);
- } else if (to == null) {
- subMap = rangeIndex.tailMap(from, fromInclusive);
- } else {
- subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive);
- }
- for (ConciseSet set : subMap.values()) {
- r.addAll(set);
- }
- return r;
- } finally {
- rwLock.readLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
deleted file mode 100644
index 5e3e771..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
-
- GTInfo info;
- GTInvertedIndex index;
- ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
- ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
- @BeforeClass
- public static void setUp() throws Exception {
- staticCreateTestMetadata();
- }
-
- @AfterClass
- public static void after() throws Exception {
- cleanAfterClass();
- }
-
- public SimpleInvertedIndexTest() {
-
- info = UnitTestSupport.advancedInfo();
- TblColRef colA = info.colRef(0);
-
- // block i contains value "i", the last is NULL
- index = new GTInvertedIndex(info);
- GTRowBlock mockBlock = GTRowBlock.allocate(info);
- GTRowBlock.Writer writer = mockBlock.getWriter();
- GTRecord record = new GTRecord(info);
- for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
- for (int j = 0; j < info.getRowBlockSize(); j++) {
- writer.append(record);
- }
- writer.readyForFlush();
- index.add(mockBlock);
-
- writer.clearForNext();
- }
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
- basicResults.add(set(9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
- basicResults.add(set(0));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
- basicResults.add(set(0, 5));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
- basicResults.add(set(0, 1, 2));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
- basicResults.add(set(0, 1, 2, 3));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
- basicResults.add(set(4, 5, 6, 7, 8));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
- basicResults.add(set(3, 4, 5, 6, 7, 8));
- }
-
- @Test
- public void testBasics() {
- for (int i = 0; i < basicFilters.size(); i++) {
- assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
- }
- }
-
- @Test
- public void testLogicalAnd() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.retainAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testLogicalOr() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.addAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testNotEvaluable() {
- ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
- assertEquals(all, index.filter(notEvaluable));
-
- LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
- assertEquals(all, index.filter(or));
-
- LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
- assertEquals(basicResults.get(0), index.filter(and));
- }
-
- public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
- CompareTupleFilter filter = new CompareTupleFilter(op);
- filter.addChild(columnFilter(col));
- for (int i : ids) {
- filter.addChild(constFilter(i));
- }
- return filter;
- }
-
- public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
- LogicalTupleFilter filter = new LogicalTupleFilter(op);
- for (TupleFilter f : filters)
- filter.addChild(f);
- return filter;
- }
-
- public static ColumnTupleFilter columnFilter(TblColRef col) {
- return new ColumnTupleFilter(col);
- }
-
- public static ConstantTupleFilter constFilter(int id) {
- byte[] space = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(space);
- StringSerializer stringSerializer = new StringSerializer(DataType.getType("string"));
- stringSerializer.serialize("" + id, buf);
- ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
- return new ConstantTupleFilter(data);
- }
-
- public static ConciseSet set(int... ints) {
- ConciseSet set = new ConciseSet();
- for (int i : ints)
- set.add(i);
- return set;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 36d34c8..9478555 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -39,6 +39,15 @@
<artifactId>kylin-core-cube</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 87c4438..0832e80 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -39,6 +39,29 @@
<artifactId>kylin-core-common</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <!-- Compiled -->
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 9bd3f04..8c3da5f 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -39,8 +39,28 @@
<artifactId>kylin-core-cube</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index be2cdf6..fe6d998 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -70,6 +70,7 @@
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Hadoop dependency -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 874ead6..02890a6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -43,7 +43,7 @@
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica</artifactId>
+ <artifactId>avatica-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -69,9 +69,7 @@
<artifactSet>
<excludes>
- <exclude>com.google.protobuf:*</exclude>
- <exclude>commons-logging:*</exclude>
- <exclude>commons-codec:*</exclude>
+ <exclude>org.slf4j:jcl-over-slf4j:*</exclude>
</excludes>
</artifactSet>
@@ -85,9 +83,17 @@
<shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>${shadeBase}.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons</shadedPattern>
+ </relocation>
</relocations>
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b82eee2..b109783 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,15 +55,32 @@
<!-- HBase versions -->
<hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+
+ <!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
- <!-- Hadoop deps, keep compatible with hadoop2.version -->
+ <!-- Spark versions -->
+ <spark.version>1.6.3</spark.version>
+ <kryo.version>4.0.0</kryo.version>
+ <reflections.version>0.9.10</reflections.version>
+
+ <!-- Calcite Version -->
+ <calcite.version>1.11.0</calcite.version>
+ <avatica.version>1.9.0</avatica.version>
+
+ <!-- Hadoop Common deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
<curator.version>2.7.1</curator.version>
- <jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
+ <jsch.version>0.1.53</jsch.version>
<commons-cli.version>1.2</commons-cli.version>
+ <commons-lang.version>2.6</commons-lang.version>
+ <commons-io.version>2.5</commons-io.version>
+ <commons-math3.version>3.6.1</commons-math3.version>
+
+ <!-- Calcite deps, keep compatible with calcite.version -->
+ <jackson.version>2.6.3</jackson.version>
<!-- Test Dependency versions -->
<antlr.version>3.4</antlr.version>
@@ -75,26 +92,16 @@
<jamm.version>0.3.1</jamm.version>
<!-- Commons -->
- <commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-collections.version>3.2.2</commons-collections.version>
- <commons-io.version>2.5</commons-io.version>
<commons-daemon.version>1.0.15</commons-daemon.version>
<commons-email.version>1.4</commons-email.version>
- <commons-math3.version>3.6.1</commons-math3.version>
-
- <!-- Spark -->
- <spark.version>1.6.3</spark.version>
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.21</slf4j.version>
- <reflections.version>0.9.10</reflections.version>
- <jsch.version>0.1.53</jsch.version>
<xerces.version>2.11.0</xerces.version>
<xalan.version>2.7.2</xalan.version>
- <extendedset.version>1.3.4</extendedset.version>
- <kryo.version>4.0.0</kryo.version>
<ehcache.version>2.10.2.2.21</ehcache.version>
<apache-httpclient.version>4.2.5</apache-httpclient.version>
<roaring.version>0.6.18</roaring.version>
@@ -111,10 +118,6 @@
<opensaml.version>2.6.4</opensaml.version>
<aspectj.version>1.8.9</aspectj.version>
- <!-- Calcite Version -->
- <calcite.version>1.11.0</calcite.version>
- <avatica.version>1.9.0</avatica.version>
-
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
<sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
@@ -390,6 +393,66 @@
<version>${hadoop2.version}</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>${jsr305.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>${jsch.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>${commons-cli.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>${commons-math3.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- HBase2 dependencies -->
<dependency>
@@ -461,6 +524,11 @@
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ <version>${avatica.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
<version>${avatica.version}</version>
</dependency>
@@ -476,23 +544,45 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo-shaded</artifactId>
+ <version>${kryo.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>${reflections.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- Kafka dependency -->
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>${kafka.version}</version>
@@ -501,26 +591,6 @@
<!-- Other dependencies -->
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- <version>${commons-cli.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${commons-lang.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
@@ -531,21 +601,15 @@
<version>${commons-collections.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>${commons-math3.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons-io.version}</version>
- </dependency>
- <dependency>
<groupId>commons-daemon</groupId>
<artifactId>commons-daemon</artifactId>
<version>${commons-daemon.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-email</artifactId>
+ <version>${commons-email.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
@@ -571,40 +635,18 @@
<version>${slf4j.version}</version>
</dependency>
+ <!-- Test -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-email</artifactId>
- <version>${commons-email.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>${jsr305.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>${reflections.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- <version>${jsch.version}</version>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.dbunit</groupId>
<artifactId>dbunit</artifactId>
<version>${dbunit.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
@@ -615,6 +657,7 @@
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>xerces</groupId>
@@ -627,31 +670,10 @@
<version>${xalan.version}</version>
</dependency>
<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
- <version>${extendedset.version}</version>
- </dependency>
- <dependency>
- <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo-shaded</artifactId>
- <version>${kryo.version}</version>
- </dependency>
- <dependency>
<groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
<version>${jamm.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 6ab74a7..d35c74c 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -47,10 +47,16 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
</dependency>
+
+ <!-- Provided -->
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 454ed6b..3c96e60 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -233,6 +233,10 @@
<artifactId>jetty-all</artifactId>
<groupId>org.eclipse.jetty.aggregate</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>calcite-core</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index e2fe448..da5e033 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -35,23 +35,19 @@
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-mr</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
+ <artifactId>kylin-engine-mr</artifactId>
</dependency>
+ <!-- Provided -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
</dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -76,9 +72,5 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
-
-
-
</dependencies>
-
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 3aea531..054b2e7 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -119,12 +119,8 @@
<include>org.apache.kylin:kylin-core-metadata</include>
<include>org.apache.kylin:kylin-core-dictionary</include>
<include>org.apache.kylin:kylin-core-cube</include>
- <include>com.ning:compress-lzf</include>
<include>org.roaringbitmap:RoaringBitmap</include>
<include>com.tdunning:t-digest</include>
- <!-- below for inverted index only -->
- <include>com.n3twork.druid:extendedset</include>
- <include>org.apache.commons:commons-lang3</include>
</includes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 91040d4..e9bb9ba 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -54,6 +54,33 @@
<artifactId>kylin-source-hive</artifactId>
</dependency>
+ <!-- As KylinConfigCLI will be called before any dependency loaded, has to define the following as compile in tool -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
<!--Env-->
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -100,6 +127,12 @@
<includes>
<!-- shade the httpcore to avoid the lower version conflict with HBase one -->
<include>org.apache.httpcomponents:httpcore</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ <include>log4j:log4j</include>
+ <include>commons-io:commons-io</include>
+ <include>commons-lang:commons-lang</include>
+ <include>com.google.guava:guava</include>
<include>org.apache.kylin:kylin-tool</include>
</includes>
</artifactSet>
[4/8] kylin git commit: KYLIN-2434 use
kylin.source.hive.database-for-flat-table in spark cubing
Posted by bi...@apache.org.
KYLIN-2434 use kylin.source.hive.database-for-flat-table in spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f62465cc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f62465cc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f62465cc
Branch: refs/heads/KYLIN-2428
Commit: f62465cc5a5fe0f90142c0606c0c75e8c721bf0e
Parents: b32cc95
Author: lidongsjtu <li...@apache.org>
Authored: Tue Feb 7 17:31:07 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Feb 7 17:31:14 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f62465cc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 8892a73..071806c 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,11 +150,12 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
setupClasspath(sc, confPath);
HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
+ final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+
HiveContext sqlContext = new HiveContext(sc.sc());
- final DataFrame intermediateTable = sqlContext.table(hiveTable);
+ final DataFrame intermediateTable = sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + hiveTable);
System.setProperty(KylinConfig.KYLIN_CONF, confPath);
- final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
final CubeDesc cubeDesc = cubeInstance.getDescriptor();
final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
[5/8] kylin git commit: KYLIN-2434 minor fix
Posted by bi...@apache.org.
KYLIN-2434 minor fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eee9ecbc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eee9ecbc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eee9ecbc
Branch: refs/heads/KYLIN-2428
Commit: eee9ecbcb031862ed316f630933bb320cf93a9d1
Parents: f62465c
Author: lidongsjtu <li...@apache.org>
Authored: Tue Feb 7 17:37:21 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Feb 7 17:37:21 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eee9ecbc/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 071806c..317d2bd 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -150,12 +150,12 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
setupClasspath(sc, confPath);
HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
+ System.setProperty(KylinConfig.KYLIN_CONF, confPath);
final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
HiveContext sqlContext = new HiveContext(sc.sc());
final DataFrame intermediateTable = sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + hiveTable);
- System.setProperty(KylinConfig.KYLIN_CONF, confPath);
final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
final CubeDesc cubeDesc = cubeInstance.getDescriptor();
final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);