You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/09/07 08:47:12 UTC
[kylin] 04/24: KYLIN-5689 fix local debug in Kylin5
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b05930e845bf8aa3c0e38324117c2f9694a6b7ba
Author: Yaguang Jia <ji...@foxmail.com>
AuthorDate: Tue May 16 14:12:16 2023 +0800
KYLIN-5689 fix local debug in Kylin5
* KYLIN-5689 allow 'dev' profile to run without sandbox, just local spark and local fs
* KYLIN-5689 add h2 dependency to profile dev
* KYLIN-5689 support mock prod metadata
---------
Co-authored-by: Li Yang <li...@apache.org>
---
.../apache/kylin/rest/KylinPrepareEnvListener.java | 10 ++-
.../org/apache/kylin/rest/config/AppConfig.java | 9 ++
.../apache/kylin/rest/config/AppInitializer.java | 12 ++-
.../apache/kylin/common/lock/LocalLockFactory.java | 95 ++++++++++++++++++++++
.../kylin/common/lock/LocalLockFactoryTest.java | 49 +++++++++++
.../test_case_data/localmeta/kylin.properties | 3 +
.../config/initialize/QueryAppInitializer.java | 2 +
src/server/pom.xml | 11 ++-
.../apache/kylin/rest/ZookeeperClusterManager.java | 2 +-
src/server/src/main/resources/application.yaml | 3 +
.../tool/security/KylinPasswordResetCLITest.java | 1 +
11 files changed, 190 insertions(+), 7 deletions(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index 5a83565d0c..9a93d3f596 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -66,7 +66,10 @@ public class KylinPrepareEnvListener implements EnvironmentPostProcessor, Ordere
setSandboxEnvs("../examples/test_case_data/sandbox");
}
} else if (env.acceptsProfiles(Profiles.of("dev"))) {
- setLocalEnvs();
+ if (env.getSystemEnvironment().containsKey(KylinConfig.KYLIN_CONF))
+ ClassUtil.addClasspath(env.getSystemEnvironment().get(KylinConfig.KYLIN_CONF).toString());
+ if (!StringUtils.equals("true", env.getProperty("dev.diag-meta")))
+ setLocalEnvs();
}
// enable CC check
Unsafe.setProperty("needCheckCC", "true");
@@ -133,7 +136,10 @@ public class KylinPrepareEnvListener implements EnvironmentPostProcessor, Ordere
// Load H2 Tables (inner join) for pushdown to rdbms in local debug mode
try {
- Connection h2Connection = DriverManager.getConnection("jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1", "sa", "");
+ String username = System.getProperty("kylin.query.pushdown.jdbc.username");
+ String password = System.getProperty("kylin.query.pushdown.jdbc.password");
+ Connection h2Connection = DriverManager.getConnection("jdbc:h2:mem:db_default;DB_CLOSE_DELAY=-1", username,
+ password);
H2Database h2DB = new H2Database(h2Connection, KylinConfig.getInstanceFromEnv(), "default");
h2DB.loadAllTables();
} catch (SQLException ex) {
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
index 45c733ac38..e4b68a1424 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppConfig.java
@@ -32,6 +32,7 @@ import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.cluster.DefaultClusterManager;
import org.apache.kylin.rest.handler.KapNoOpResponseErrorHandler;
import org.apache.kylin.rest.interceptor.ReloadAuthoritiesInterceptor;
+import org.apache.kylin.rest.service.QuerySmartSupporter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -148,6 +149,14 @@ public class AppConfig implements WebMvcConfigurer {
return new DefaultClusterManager(port);
}
+ @Bean
+ @ConditionalOnMissingBean(QuerySmartSupporter.class)
+ public QuerySmartSupporter querySmartSupporter() {
+ return (project, queries, manual) -> {
+ // noop
+ };
+ }
+
@Bean(name = "normalRestTemplate")
public RestTemplate restTemplate() {
RestTemplate restTemplate = new RestTemplate();
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
index b34dcb6b2b..f36aa23f26 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
@@ -107,7 +107,9 @@ public class AppInitializer {
String localIdentify = EpochOrchestrator.getOwnerIdentity().split("\\|")[0];
return localIdentify.equalsIgnoreCase(instance);
});
- streamingJobStatsStore = new JdbcStreamingJobStatsStore(kylinConfig);
+
+ if (kylinConfig.streamingEnabled())
+ streamingJobStatsStore = new JdbcStreamingJobStatsStore(kylinConfig);
// register scheduler listener
EventBusFactory.getInstance().register(new JobSchedulerListener(), false);
@@ -156,7 +158,11 @@ public class AppInitializer {
EventBusFactory.getInstance().register(new UserAclListener(), true);
postInit();
- log.info("Application metadata initialization is complete.");
+
+ log.info("Kylin initialization completed.");
+ log.info("KylinConfig in env, ID is {}", kylinConfig.hashCode());
+ log.info("KylinConfig in env, metadata is {}", kylinConfig.getMetadataUrl());
+ log.info("KylinConfig in env, working dir is {}", kylinConfig.getHdfsWorkingDirectory());
}
private void warmUpSystemCache() {
@@ -170,7 +176,7 @@ public class AppInitializer {
}
@EventListener(ApplicationReadyEvent.class)
- public void afterReady(ApplicationReadyEvent event) {
+ public void afterReady(ApplicationReadyEvent ignoredEvent) {
val kylinConfig = KylinConfig.getInstanceFromEnv();
setFsUrlStreamHandlerFactory();
if (kylinConfig.isJobNode()) {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/LocalLockFactory.java b/src/core-common/src/main/java/org/apache/kylin/common/lock/LocalLockFactory.java
new file mode 100644
index 0000000000..5d95284146
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/lock/LocalLockFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.lock;
+
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.HashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A local version of the distributed lock factory.
+ * Wraps around the ReentrantLock, a straightforward implementation.
+ */
+@Slf4j
+public class LocalLockFactory extends DistributedLockFactory {
+
+ public Lock getLockForClient(String client, String lockId) {
+ return getLockInternal(client, lockId);
+ }
+
+ private static final HashMap<String, ReentrantLock> cache = new HashMap<>();
+ private Lock getLockInternal(String client, String lockId) {
+ synchronized (cache) {
+ cache.putIfAbsent(lockId, new ReentrantLock());
+ ReentrantLock lock = cache.get(lockId);
+ return new LocalLock(lock, client, lockId);
+ }
+ }
+
+ public void initialize() {
+ // nothing to do
+ }
+
+ @AllArgsConstructor
+ private static class LocalLock implements Lock {
+ private ReentrantLock lock;
+ private String client;
+ private String lockId;
+
+ @Override
+ public void lock() {
+ log.debug(client + " locking " + lockId);
+ lock.lock();
+ }
+
+ @Override
+ public void lockInterruptibly() throws InterruptedException {
+ lock.lockInterruptibly();
+ }
+
+ @Override
+ public boolean tryLock() {
+ return lock.tryLock();
+ }
+
+ @Override
+ public boolean tryLock(long l, TimeUnit timeUnit) throws InterruptedException {
+ return lock.tryLock(l, timeUnit);
+ }
+
+ @Override
+ public void unlock() {
+ synchronized(cache) {
+ log.debug(client + " unlocking " + lockId);
+ lock.unlock();
+ cache.remove(lockId);
+ }
+ }
+
+ @Override
+ public Condition newCondition() {
+ return lock.newCondition();
+ }
+ }
+}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/LocalLockFactoryTest.java b/src/core-common/src/test/java/org/apache/kylin/common/lock/LocalLockFactoryTest.java
new file mode 100644
index 0000000000..20bd93d2f8
--- /dev/null
+++ b/src/core-common/src/test/java/org/apache/kylin/common/lock/LocalLockFactoryTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.lock;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
+import org.junit.jupiter.api.Test;
+
+class LocalLockFactoryTest {
+
+ @Test
+ void testLocking() throws InterruptedException {
+
+ LocalLockFactory lockFactory = new LocalLockFactory();
+ lockFactory.initialize();
+
+ String client = "client1";
+ String lockId = "lock1";
+
+ Lock lock1 = lockFactory.getLockForClient(client, lockId);
+ lock1.lock();
+ lock1.unlock();
+
+ assertTrue(lock1.tryLock());
+ assertTrue(lock1.tryLock(30, TimeUnit.SECONDS));
+
+ lock1.lockInterruptibly();
+ lock1.newCondition();
+ }
+}
diff --git a/src/examples/test_case_data/localmeta/kylin.properties b/src/examples/test_case_data/localmeta/kylin.properties
index 239cfe4b0c..9c5e1811b4 100755
--- a/src/examples/test_case_data/localmeta/kylin.properties
+++ b/src/examples/test_case_data/localmeta/kylin.properties
@@ -19,6 +19,9 @@
kylin.env=UT
kylin.env.zookeeper-connect-string=localhost:2181
+# The following two parameters are configured for use during local development.
+# kylin.metadata.url=${KYLIN_METADATA_URL}
+# kylin.metadata.distributed-lock-impl=org.apache.kylin.common.lock.LocalLockFactory
### SERVICE ###
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryAppInitializer.java b/src/query-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryAppInitializer.java
index 400cfbc19a..972fab9324 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryAppInitializer.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryAppInitializer.java
@@ -21,12 +21,14 @@ import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.rest.service.QueryHistoryScheduler;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
+@Profile("!dev")
public class QueryAppInitializer implements InitializingBean {
@Autowired
diff --git a/src/server/pom.xml b/src/server/pom.xml
index beeaf65275..9a5e033725 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -258,7 +258,6 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
- <scope>test</scope>
</dependency>
<!-- mysql -->
@@ -492,6 +491,16 @@
</pluginRepository>
</pluginRepositories>
<profiles>
+ <profile>
+ <id>dev</id>
+ <dependencies>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>sandbox</id>
<dependencies>
diff --git a/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java b/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java
index 29b286fce4..50761c7b2b 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/ZookeeperClusterManager.java
@@ -36,7 +36,7 @@ import org.springframework.stereotype.Component;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
-@ConditionalOnZookeeperEnabled
+@ConditionalOnZookeeperEnabled // if missing, a default impl will be filled by AppConfig.clusterManager()
@Component
@Slf4j
public class ZookeeperClusterManager implements ClusterManager {
diff --git a/src/server/src/main/resources/application.yaml b/src/server/src/main/resources/application.yaml
index edd4fe5530..8f7ba16fd8 100644
--- a/src/server/src/main/resources/application.yaml
+++ b/src/server/src/main/resources/application.yaml
@@ -121,6 +121,9 @@ spring:
cloud:
zookeeper:
enabled: false
+ web:
+ resources:
+ static-locations: "file://${KYLIN_HOME}/kystudio/dist"
---
spring:
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java b/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java
index 5b067f91fb..e3c17825cc 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java
@@ -66,6 +66,7 @@ public class KylinPasswordResetCLITest extends LogOutputTestCase {
@Test
public void testResetAdminPassword() throws Exception {
val pwdEncoder = new BCryptPasswordEncoder();
+ overwriteSystemProp("kylin.metadata.random-admin-password.enabled", "true");
overwriteSystemProp("kylin.security.user-password-encoder", pwdEncoder.getClass().getName());
overwriteSystemProp("kylin.metadata.random-admin-password.enabled", "true");
val user = new ManagedUser("ADMIN", "KYLIN", true, Constant.ROLE_ADMIN, Constant.GROUP_ALL_USERS);