You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/08 06:03:44 UTC
[7/8] kylin git commit: KYLIN-2388 Hot load kylin config from web
KYLIN-2388 Hot load kylin config from web
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39afa519
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39afa519
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39afa519
Branch: refs/heads/KYLIN-2428
Commit: 39afa5197e197d5143843a3bb62c528749b03418
Parents: 7b860ad
Author: kangkaisen <ka...@live.com>
Authored: Thu Jan 12 14:35:59 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Tue Feb 7 19:57:29 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 6 +-
.../apache/kylin/common/KylinConfigTest.java | 28 ++++-----
.../util/HotLoadKylinPropertiesTestCase.java | 64 ++++++++++++++++++++
.../kylin/cube/CubeSpecificConfigTest.java | 37 ++++++-----
.../apache/kylin/job/JobEngineConfigTest.java | 47 ++++++++++++++
.../kylin/rest/controller/CacheController.java | 9 ++-
webapp/app/js/controllers/admin.js | 26 ++++++++
webapp/app/js/services/cache.js | 3 +-
webapp/app/partials/admin/admin.html | 3 +
9 files changed, 191 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0f40654..c7a18c6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -201,7 +201,7 @@ public class KylinConfig extends KylinConfigBase {
return kylinHome + File.separator + "conf";
}
- static File getKylinPropertiesFile() {
+ public static File getKylinPropertiesFile() {
String kylinConfHome = System.getProperty(KYLIN_CONF);
if (!StringUtils.isEmpty(kylinConfHome)) {
logger.info("Use KYLIN_CONF=" + kylinConfHome);
@@ -385,4 +385,8 @@ public class KylinConfig extends KylinConfigBase {
// }
// logger.info(buf.toString());
}
+
+ public synchronized void hotLoadKylinProperties() {
+ reloadKylinConfig(getKylinProperties());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index a426fc6..4d5f130 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -26,24 +26,12 @@ import static org.junit.Assert.assertTrue;
import java.util.Map;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.junit.Test;
import com.google.common.collect.Maps;
-public class KylinConfigTest extends LocalFileMetadataTestCase {
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
+public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
@Test
public void testMRConfigOverride() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -78,9 +66,19 @@ public class KylinConfigTest extends LocalFileMetadataTestCase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
Map<String, String> override = Maps.newHashMap();
KylinConfig configExt = KylinConfigExt.createInstance(config, override);
-
assertTrue(config.properties == configExt.properties);
config.setProperty("1234", "1234");
assertEquals("1234", configExt.getOptional("1234"));
}
+
+ @Test
+ public void testPropertiesHotLoad() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ assertEquals("whoami@kylin.apache.org", config.getKylinOwner());
+
+ updateProperty("kylin.storage.hbase.owner-tag", "kylin@kylin.apache.org");
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+ assertEquals("kylin@kylin.apache.org", config.getKylinOwner());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
new file mode 100644
index 0000000..9f5b278
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.apache.kylin.common.KylinConfig;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.Properties;
+
+/**
+ * @author kangkaisen
+ */
+
+public class HotLoadKylinPropertiesTestCase extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ protected void updateProperty(String key, String value) {
+ File propFile = KylinConfig.getKylinPropertiesFile();
+ Properties conf = new Properties();
+
+ //load
+ try (FileInputStream is = new FileInputStream(propFile)) {
+ conf.load(is);
+ conf.setProperty(key, value);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ }
+
+ //store
+ try (FileOutputStream out = new FileOutputStream(propFile)) {
+ conf.store(out, null);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
index c61f07f..17c02cc 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
@@ -21,24 +21,13 @@ package org.apache.kylin.cube;
import static org.junit.Assert.assertEquals;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.apache.kylin.cube.model.CubeDesc;
-import org.junit.After;
-import org.junit.Before;
import org.junit.Test;
-public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
+import java.io.IOException;
+public class CubeSpecificConfigTest extends HotLoadKylinPropertiesTestCase {
@Test
public void test() {
KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
@@ -57,4 +46,24 @@ public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
assertEquals("snappy", base.getHbaseDefaultCompressionCodec());
assertEquals("lz4", override.getHbaseDefaultCompressionCodec());
}
+
+ @Test
+ public void testPropertiesHotLoad() throws IOException {
+ KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+ KylinConfig oldCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig();
+ assertEquals(10, oldCubeDescConfig.getMaxConcurrentJobLimit());
+
+ //hot load Properties
+ updateProperty("kylin.job.max-concurrent-jobs", "20");
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+ CubeDescManager.getInstance(baseConfig).reloadCubeDescLocal("ssb");
+
+ //test cubeDescConfig
+ KylinConfig newCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig();
+ assertEquals(20, newCubeDescConfig.getMaxConcurrentJobLimit());
+
+ //test cubeConfig
+ KylinConfig newCubeConfig = CubeManager.getInstance(baseConfig).getCube("ssb").getConfig();
+ assertEquals(20, newCubeConfig.getMaxConcurrentJobLimit());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java
new file mode 100644
index 0000000..77914ef
--- /dev/null
+++ b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author kangkaisen
+ */
+
+public class JobEngineConfigTest extends HotLoadKylinPropertiesTestCase {
+
+ @Test
+ public void testPropertiesHotLoad() throws IOException {
+ KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(baseConfig);
+ assertEquals(10, jobEngineConfig.getMaxConcurrentJobLimit());
+
+ updateProperty("kylin.job.max-concurrent-jobs", "20");
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+
+ assertEquals(20, jobEngineConfig.getMaxConcurrentJobLimit());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
index 254aabf..8d5f00e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.rest.service.CacheService;
import org.slf4j.Logger;
@@ -39,7 +40,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
@Controller
@RequestMapping(value = "/cache")
public class CacheController extends BasicController {
-
+
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(CacheController.class);
@@ -64,6 +65,12 @@ public class CacheController extends BasicController {
cacheService.notifyMetadataChange(entity, Broadcaster.Event.getEvent(event), cacheKey);
}
+ @RequestMapping(value = "/announce/config", method = { RequestMethod.POST })
+ public void hotLoadKylinConfig() throws IOException {
+ KylinConfig.getInstanceFromEnv().hotLoadKylinProperties();
+ cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL);
+ }
+
public void setCacheService(CacheService cacheService) {
this.cacheService = cacheService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/js/controllers/admin.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/admin.js b/webapp/app/js/controllers/admin.js
index 0d36e0d..783ab17 100644
--- a/webapp/app/js/controllers/admin.js
+++ b/webapp/app/js/controllers/admin.js
@@ -58,6 +58,32 @@ KylinApp.controller('AdminCtrl', function ($scope, AdminService, CacheService, T
});
}
+ $scope.reloadConfig = function () {
+ SweetAlert.swal({
+ title: '',
+ text: 'Are you sure to reload config',
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function (isConfirm) {
+ if (isConfirm) {
+ CacheService.reloadConfig({}, function () {
+ SweetAlert.swal('Success!', 'config reload successfully', 'success');
+ }, function (e) {
+ if (e.data && e.data.exception) {
+ var message = e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ } else {
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
+ }
+ });
+ }
+
$scope.reloadMeta = function () {
SweetAlert.swal({
title: '',
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/js/services/cache.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/cache.js b/webapp/app/js/services/cache.js
index bcec603..38dc0b8 100644
--- a/webapp/app/js/services/cache.js
+++ b/webapp/app/js/services/cache.js
@@ -18,6 +18,7 @@
KylinApp.factory('CacheService', ['$resource', function ($resource, config) {
return $resource(Config.service.url + 'cache/announce/:type/:name/:action', {}, {
- clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false}
+ clean: {method: 'PUT', params: {type: 'all', name: 'all', action: 'update'}, isArray: false},
+ reloadConfig: {method: 'POST', params: {type: 'config'}, isArray: false}
});
}]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/39afa519/webapp/app/partials/admin/admin.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/admin/admin.html b/webapp/app/partials/admin/admin.html
index f2be3d9..b4bca8d 100644
--- a/webapp/app/partials/admin/admin.html
+++ b/webapp/app/partials/admin/admin.html
@@ -58,6 +58,9 @@
<a class="btn btn-primary btn-lg btn-block" tooltip="Update Server Config" class="btn btn-primary btn-lg" ng-click="toSetConfig()">Set Config</a>
</div>
<div style="padding-top: 10px;width: 260px;">
+ <a class="btn btn-primary btn-lg btn-block" tooltip="Reload Server Config" class="btn btn-primary btn-lg" ng-click="reloadConfig()">Reload Config</a>
+ </div>
+ <div style="padding-top: 10px;width: 260px;">
<a ng-click="downloadBadQueryFiles();" tooltip="Download Diagnosis Info For Current Project" class="btn btn-primary btn-lg btn-block"><i class="fa fa-ambulance"></i> Diagnosis</a>
</div>