You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/08/23 18:34:39 UTC
[kylin] 02/04: KYLIN-4705 add rest api for sparder-context url
monitor
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 14b575083a06aaaa44a722a966cdd93f8d7c1e46
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Tue Aug 18 23:27:07 2020 +0800
KYLIN-4705 add rest api for sparder-context url monitor
---
.../java/org/apache/kylin/cube/model/CubeDesc.java | 24 +++++++---------------
.../org/apache/spark/sql/SparderContext.scala | 14 ++++++++++---
.../kylin/rest/controller/AdminController.java | 9 ++++++++
3 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index b8890cb..84048e9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -183,9 +183,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonProperty("retention_range")
private long retentionRange = 0;
@JsonProperty("engine_type")
- private int engineType = IEngineAware.ID_MR_V2;
+ private int engineType = IEngineAware.ID_SPARK_II;
@JsonProperty("storage_type")
- private int storageType = IStorageAware.ID_SHARDED_HBASE;
+ private int storageType = IStorageAware.ID_PARQUET;
@JsonProperty("override_kylin_properties")
private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>();
@@ -662,10 +662,11 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
throw new RuntimeException("Error during adapting hbase mapping", e);
}
} else {
- if (hbaseMapping != null) {
- hbaseMapping.init(this);
- initMeasureReferenceToColumnFamily();
- }
+ // skip at Kylin 4.0
+// if (hbaseMapping != null) {
+// hbaseMapping.init(this);
+// initMeasureReferenceToColumnFamily();
+// }
}
// check all dimension columns are presented on rowkey
@@ -1517,17 +1518,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return globalDictCols;
}
- public boolean isShrunkenDictFromGlobalEnabled() {
- boolean needShrunkenDict = config.isShrunkenDictFromGlobalEnabled() && !getAllGlobalDictColumns().isEmpty();
- boolean needMrHiveDict = config.getMrHiveDictColumns().length > 0;
- if (needMrHiveDict && needShrunkenDict) {
- logger.info("ShrunkenDict cannot work with MrHiveDict, so shutdown ShrunkenDict.");
- return false;
- } else {
- return needShrunkenDict;
- }
- }
-
// UHC (ultra high cardinality column): contain the ShardByColumns and the GlobalDictionaryColumns
public List<TblColRef> getAllUHCColumns() {
List<TblColRef> uhcColumns = new ArrayList<>();
diff --git a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
index f70ea67..1c9f500 100644
--- a/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
+++ b/kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/SparderContext.scala
@@ -44,7 +44,7 @@ object SparderContext extends Logging {
private var initializingThread: Thread = null
@volatile
- var APP_MASTER_TRACK_URL: String = null
+ var master_app_url: String = _
def getSparkSession: SparkSession = withClassLoad {
if (spark == null || spark.sparkContext.isStopped) {
@@ -60,7 +60,14 @@ object SparderContext extends Logging {
}
def setAPPMasterTrackURL(url: String): Unit = {
- APP_MASTER_TRACK_URL = url
+ master_app_url = url
+ }
+
+ def appMasterTrackURL(): String = {
+ if (master_app_url == null)
+ master_app_url
+ else
+ "Not_initialized"
}
def isSparkAvailable: Boolean = {
@@ -139,6 +146,7 @@ object SparderContext extends Logging {
.getOrCreateKylinSession()
}
spark = sparkSession
+ sparkSession.sparkContext.applicationId
logInfo("Spark context started successfully with stack trace:")
logInfo(Thread.currentThread().getStackTrace.mkString("\n"))
logInfo(
@@ -147,7 +155,7 @@ object SparderContext extends Logging {
.getContextClassLoader
.toString)
initMonitorEnv()
- APP_MASTER_TRACK_URL = null
+ master_app_url = null
} catch {
case throwable: Throwable =>
logError("Error for initializing spark ", throwable)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
index 843c609..cb44c0a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java
@@ -33,6 +33,7 @@ import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.service.AdminService;
import org.apache.kylin.rest.service.CubeService;
+import org.apache.spark.sql.SparderContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Controller;
@@ -106,6 +107,14 @@ public class AdminController extends BasicController {
return configRes;
}
+ @RequestMapping(value = "/sparder_rul", method = { RequestMethod.GET }, produces = { "application/json" })
+ @ResponseBody
+ public GeneralResponse getSparderUrl() throws IOException {
+ GeneralResponse configRes = new GeneralResponse();
+ configRes.put("url", SparderContext.appMasterTrackURL());
+ return configRes;
+ }
+
@RequestMapping(value = "/metrics/cubes", method = { RequestMethod.GET }, produces = { "application/json" })
@ResponseBody
public MetricsResponse cubeMetrics(MetricsRequest request) {