You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/01/29 09:31:48 UTC
[kylin] 06/06: KYLIN-5398 Fix the problem of code override caused by revert
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ee2bfd18a2d679a0f485bd0db9bac84f95604363
Author: Guoliang Sun <gu...@kyligence.io>
AuthorDate: Tue Nov 15 11:24:39 2022 +0800
KYLIN-5398 Fix the problem of code override caused by revert
---
dev-support/sandbox/conf/kylin.properties | 4 ++++
src/examples/test_case_data/sandbox/kylin.properties | 4 ++++
.../kylin/query/asyncprofiler/AsyncProfilingTest.scala | 2 ++
.../QueryAsyncProfilerDriverPluginTest.scala | 8 ++++----
.../kylin/common/asyncprofiler/AsyncArchUtil.java | 2 +-
.../kylin/common/asyncprofiler/AsyncProfiler.java | 9 ++++-----
.../async-profiler-lib/linux64/libasyncProfiler.so | Bin 314098 -> 0 bytes
.../async-profiler-lib/macOS/libasyncProfiler.so | Bin 239460 -> 0 bytes
.../kylin/common/asyncprofiler/AsyncArchUtilTest.java | 2 +-
.../kylin/common/asyncprofiler/AsyncProfilerTest.java | 3 +--
.../BuildAsyncProfilerDriverPluginTest.scala | 4 ++--
11 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/dev-support/sandbox/conf/kylin.properties b/dev-support/sandbox/conf/kylin.properties
index 56d8d99730..7ca8e1ee4f 100644
--- a/dev-support/sandbox/conf/kylin.properties
+++ b/dev-support/sandbox/conf/kylin.properties
@@ -139,6 +139,10 @@ kylin.security.saml.context-path=/kylin
kylin.influxdb.address=influxdb:8086
kylin.swagger.enabled=true
+### Flame Graph
+kylin.query.async-profiler-enabled=false
+kylin.engine.async-profiler-enabled=false
+
# for tests
kylin.test.bcc.new-key=some-value
kylin.engine.mr.config-override.test1=test1
diff --git a/src/examples/test_case_data/sandbox/kylin.properties b/src/examples/test_case_data/sandbox/kylin.properties
index 73426da7ae..5fe04b2fe6 100644
--- a/src/examples/test_case_data/sandbox/kylin.properties
+++ b/src/examples/test_case_data/sandbox/kylin.properties
@@ -127,6 +127,10 @@ kylin.security.saml.context-path=/kylin
kylin.influxdb.address=sandbox:8086
kylin.swagger.enabled=true
+### Flame Graph
+kylin.query.async-profiler-enabled=false
+kylin.engine.async-profiler-enabled=false
+
# for tests
kylin.test.bcc.new-key=some-value
kylin.engine.mr.config-override.test1=test1
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
index 21c08c4ee7..33f497ee33 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
@@ -40,7 +40,9 @@ class AsyncProfilingTest extends AsyncPluginWithMeta {
.setAppName(getClass.getName)
.set(SparkLauncher.SPARK_MASTER, "local[1]")
.set("spark.plugins", sparkPluginName)
+ .set("spark.submit.deployMode", "client")
+ System.setProperty("spark.profiler.flagsDir", flagFileDir)
sc = new SparkContext(conf)
}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
index 8d82c3c684..44c564b3ba 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
@@ -37,12 +37,12 @@ class QueryAsyncProfilerDriverPluginTest extends AsyncPluginWithMeta {
}
test("plugin initialization") {
- Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
+ Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString)
new QueryAsyncProfilerDriverPlugin().receive("NEX-1:start,event=cpu")
}
test("plugin initialization receive result") {
- Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
+ Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString)
try {
new QueryAsyncProfilerDriverPlugin().receive("RES-1:flamegraph")
} catch {
@@ -51,11 +51,11 @@ class QueryAsyncProfilerDriverPluginTest extends AsyncPluginWithMeta {
}
test("plugin initialization receive others") {
- Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
+ Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString)
try {
new QueryAsyncProfilerDriverPlugin().receive("OTH-1:start,event=cpu")
} catch {
case _: Throwable =>
}
}
-}
+}
\ No newline at end of file
diff --git a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
index a1b77ab622..17b9904ff1 100644
--- a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
@@ -74,4 +74,4 @@ public class AsyncArchUtil {
private static void addProcessors(ArchType archType, final String... keys) {
Stream.of(keys).forEach(key -> ARCH_TO_PROCESSOR.put(key, archType));
}
-}
\ No newline at end of file
+}
diff --git a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
index 1c0b3a795e..4f7ab73de6 100644
--- a/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
@@ -18,6 +18,10 @@
package org.apache.kylin.common.asyncprofiler;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_ARM64;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_X64;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_MAC;
+
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -31,12 +35,7 @@ public class AsyncProfiler {
private static final Logger logger = LoggerFactory.getLogger(AsyncProfiler.class);
- // async profiler native files
- public static final String ASYNC_PROFILER_LIB_MAC = "libasyncProfiler-mac.so";
- public static final String ASYNC_PROFILER_LIB_LINUX_X64 = "libasyncProfiler-linux-x64.so";
- public static final String ASYNC_PROFILER_LIB_LINUX_ARM64 = "libasyncProfiler-linux-arm64.so";
private static final String LIB_PARENT = "/async-profiler-lib/";
-
private static AsyncProfiler profiler;
private boolean loaded = false;
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so
deleted file mode 100755
index 4153f52868..0000000000
Binary files a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so and /dev/null differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so
deleted file mode 100755
index c7298c6b82..0000000000
Binary files a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so and /dev/null differ
diff --git a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
index 0ba16c4a75..b2a3a26044 100644
--- a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
@@ -37,4 +37,4 @@ public class AsyncArchUtilTest {
archType = AsyncArchUtil.getProcessor("aarch64");
Assert.assertEquals(AsyncArchUtil.ArchType.LINUX_ARM64, archType);
}
-}
\ No newline at end of file
+}
diff --git a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
index c819afc4f5..42cebf7eef 100644
--- a/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
@@ -66,5 +66,4 @@ public class AsyncProfilerTest {
Assert.assertThrows("Profiler is not active", IllegalStateException.class,
AsyncProfiler.getInstance(true)::stop);
}
-
-}
+}
\ No newline at end of file
diff --git a/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala b/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala
index 6a23c9e154..63fd2ffbfe 100644
--- a/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala
+++ b/src/spark-project/spark-common/src/test/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPluginTest.scala
@@ -75,7 +75,7 @@ class BuildAsyncProfilerDriverPluginTest extends SparkFunSuite with BeforeAndAft
.set("spark.plugins", sparkPluginName)
sc = new SparkContext(conf)
- Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
+ Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString)
sc.stop()
sc = null
@@ -89,7 +89,7 @@ class BuildAsyncProfilerDriverPluginTest extends SparkFunSuite with BeforeAndAft
System.setProperty("spark.profiler.flagsDir", flagFileDir)
sc = new SparkContext(conf)
- Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
+ Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins").toString)
sc.stop()
sc = null