You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/20 07:33:44 UTC
[kylin] 01/02: KYLIN-3415 Remove external module
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f77c7ba340189de3cdc4c761770a85bcdd1dd187
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Jun 19 18:05:08 2018 +0800
KYLIN-3415 Remove external module
---
core-common/pom.xml | 6 --
.../org/apache/kylin/common/htrace/HtraceInit.java | 94 ----------------------
.../kylin/storage/gtrecord/ScannerWorker.java | 33 ++++----
pom.xml | 1 -
.../kylin/query/enumerator/OLAPEnumerator.java | 30 +++----
.../apache/kylin/rest/service/QueryService.java | 48 -----------
6 files changed, 27 insertions(+), 185 deletions(-)
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 47bdab9..8dca01f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -67,12 +67,6 @@
<artifactId>jsch</artifactId>
</dependency>
- <!-- Metric & Tracing -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-external-htrace</artifactId>
- </dependency>
-
<!-- Env & Test -->
<dependency>
<groupId>junit</groupId>
diff --git a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java b/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
deleted file mode 100644
index c685947..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.htrace;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.HTraceConfiguration;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.SpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.LocalFileSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.StandardOutSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.ZipkinSpanReceiver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
-public class HtraceInit {
- public static final Logger logger = LoggerFactory.getLogger(HtraceInit.class);
-
- static {
- try {
- // init for HTrace
- String fileName = System.getProperty("spanFile");
-
- Collection<SpanReceiver> rcvrs = new HashSet<SpanReceiver>();
-
- // writes spans to a file if one is provided to maven with
- // -DspanFile="FILENAME", otherwise writes to standard out.
- if (fileName != null) {
- File f = new File(fileName);
- File parent = f.getParentFile();
- if (parent != null && !parent.exists() && !parent.mkdirs()) {
- throw new IllegalArgumentException("Couldn't create file: " + fileName);
- }
- HashMap<String, String> conf = new HashMap<String, String>();
- conf.put("local-file-span-receiver.path", fileName);
- LocalFileSpanReceiver receiver = new LocalFileSpanReceiver(HTraceConfiguration.fromMap(conf));
- rcvrs.add(receiver);
- } else {
- rcvrs.add(new StandardOutSpanReceiver(HTraceConfiguration.EMPTY));
- }
-
- String hostKey = "zipkin.collector-hostname";
- String host = System.getProperty(hostKey);
- String portKey = "zipkin.collector-port";
- String port = System.getProperty(portKey);
-
- Map<String, String> confMap = Maps.newHashMap();
- if (!StringUtils.isEmpty(host)) {
- confMap.put(hostKey, host);
- logger.info("{} is set to {}", hostKey, host);
- }
- if (!StringUtils.isEmpty(port)) {
- confMap.put(portKey, port);
- logger.info("{} is set to {}", portKey, port);
- }
-
- ZipkinSpanReceiver zipkinSpanReceiver = new ZipkinSpanReceiver(HTraceConfiguration.fromMap(confMap));
- rcvrs.add(zipkinSpanReceiver);
-
- for (SpanReceiver receiver : rcvrs) {
- Trace.addReceiver(receiver);
- logger.info("SpanReceiver {} added.", receiver);
- }
- } catch (Exception e) {
- //
- logger.error("Failed to init HTrace", e);
- }
- }
-
- public static void init() { //only to trigger static block
- }
-}
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
index def7e58..0a445fe 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -30,8 +30,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStorage;
import org.apache.kylin.metadata.model.ISegment;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,26 +46,23 @@ public class ScannerWorker {
inputArgs = new Object[] { segment, cuboid, scanRequest, gtStorage, context };
- try (TraceScope scope = Trace.startSpan("visit segment " + segment.getName())) {
-
- if (scanRequest == null) {
- logger.info("Segment {} will be skipped", segment);
- internal = new EmptyGTScanner();
- return;
- }
+ if (scanRequest == null) {
+ logger.info("Segment {} will be skipped", segment);
+ internal = new EmptyGTScanner();
+ return;
+ }
- final GTInfo info = scanRequest.getInfo();
+ final GTInfo info = scanRequest.getInfo();
- try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
- .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
- .newInstance(segment, cuboid, info, context); // default behavior
- internal = rpc.getGTScanner(scanRequest);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ try {
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
+ .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
+ .newInstance(segment, cuboid, info, context); // default behavior
+ internal = rpc.getGTScanner(scanRequest);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
-
+
checkNPE();
}
diff --git a/pom.xml b/pom.xml
index 85b6b67..596f798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1163,7 +1163,6 @@
</build>
<modules>
- <module>external</module>
<module>atopcalcite</module>
<module>core-common</module>
<module>core-metadata</module>
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 721e00a..3f7beff 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -26,8 +26,6 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.StorageFactory;
import org.slf4j.Logger;
@@ -103,25 +101,23 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
}
private ITupleIterator queryStorage() {
- try (TraceScope scope = Trace.startSpan("query realization " + olapContext.realization.getCanonicalName())) {
- logger.debug("query storage...");
- // bind dynamic variables
- olapContext.bindVariable(optiqContext);
+ logger.debug("query storage...");
+ // bind dynamic variables
+ olapContext.bindVariable(optiqContext);
- olapContext.resetSQLDigest();
- SQLDigest sqlDigest = olapContext.getSQLDigest();
+ olapContext.resetSQLDigest();
+ SQLDigest sqlDigest = olapContext.getSQLDigest();
- // query storage engine
- IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
- ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
- olapContext.returnTupleInfo);
- if (logger.isDebugEnabled()) {
- logger.debug("return TupleIterator...");
- }
-
- return iterator;
+ // query storage engine
+ IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
+ ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
+ olapContext.returnTupleInfo);
+ if (logger.isDebugEnabled()) {
+ logger.debug("return TupleIterator...");
}
+
+ return iterator;
}
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 5f67aa1..b71f6ad 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -41,7 +41,6 @@ import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryContextFacade;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
-import org.apache.kylin.common.htrace.HtraceInit;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
@@ -86,9 +85,6 @@ import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
import org.apache.kylin.rest.util.TableauInterceptor;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Sampler;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -103,7 +99,6 @@ import javax.annotation.PostConstruct;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -412,14 +407,6 @@ public class QueryService extends BasicService {
final QueryContext queryContext = QueryContextFacade.current();
- TraceScope scope = null;
- if (kylinConfig.isHtraceTracingEveryQuery() || BackdoorToggles.getHtraceEnabled()) {
- logger.info("Current query is under tracing");
- HtraceInit.init();
- scope = Trace.startSpan("query life cycle for " + queryContext.getQueryId(), Sampler.ALWAYS);
- }
- String traceUrl = getTraceUrl(scope);
-
try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId())) {
SQLResponse sqlResponse = null;
String sql = sqlRequest.getSql();
@@ -446,7 +433,6 @@ public class QueryService extends BasicService {
if (sqlResponse == null && isQueryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
- Trace.addTimelineAnnotation("query cache searched");
}
// real execution if required
@@ -454,8 +440,6 @@ public class QueryService extends BasicService {
try (QueryRequestLimits limit = new QueryRequestLimits(sqlRequest.getProject())) {
sqlResponse = queryAndUpdateCache(sqlRequest, isQueryCacheEnabled);
}
- } else {
- Trace.addTimelineAnnotation("response without real execution");
}
// check authorization before return, since the response may come from cache
@@ -463,7 +447,6 @@ public class QueryService extends BasicService {
checkQueryAuth(sqlResponse, project);
sqlResponse.setDuration(queryContext.getAccumulatedMillis());
- sqlResponse.setTraceUrl(traceUrl);
logQuery(queryContext.getQueryId(), sqlRequest, sqlResponse);
try {
recordMetric(sqlRequest, sqlResponse);
@@ -478,9 +461,6 @@ public class QueryService extends BasicService {
} finally {
BackdoorToggles.cleanToggles();
QueryContextFacade.resetCurrent();
- if (scope != null) {
- scope.close();
- }
}
}
@@ -494,10 +474,8 @@ public class QueryService extends BasicService {
final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
if (isSelect) {
sqlResponse = query(sqlRequest, queryContext.getQueryId());
- Trace.addTimelineAnnotation("query almost done");
} else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled()) {
sqlResponse = update(sqlRequest);
- Trace.addTimelineAnnotation("update query almost done");
} else {
logger.debug("Directly return exception as the sql is unsupported, and query pushdown is disabled");
throw new BadRequestException(msg.getNOT_SUPPORTED_SQL());
@@ -528,7 +506,6 @@ public class QueryService extends BasicService {
kylinConfig.getLargeQueryThreshold())) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
- Trace.addTimelineAnnotation("response from execution");
} catch (Throwable e) { // calcite may throw AssertError
queryContext.stop(e);
@@ -546,7 +523,6 @@ public class QueryService extends BasicService {
Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
- Trace.addTimelineAnnotation("error response");
}
return sqlResponse;
}
@@ -561,29 +537,6 @@ public class QueryService extends BasicService {
QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
}
- private String getTraceUrl(TraceScope scope) {
- if (scope == null) {
- return null;
- }
-
- String hostname = System.getProperty("zipkin.collector-hostname");
- if (StringUtils.isEmpty(hostname)) {
- try {
- hostname = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- logger.debug("failed to get trace url due to " + e.getMessage());
- return null;
- }
- }
-
- String port = System.getProperty("zipkin.web-ui-port");
- if (StringUtils.isEmpty(port)) {
- port = "9411";
- }
-
- return "http://" + hostname + ":" + port + "/zipkin/traces/" + Long.toHexString(scope.getSpan().getTraceId());
- }
-
private String getUserName() {
String username = SecurityContextHolder.getContext().getAuthentication().getName();
if (StringUtils.isEmpty(username)) {
@@ -648,7 +601,6 @@ public class QueryService extends BasicService {
//CAUTION: should not change sqlRequest content!
//sqlRequest.setSql(correctedSql);
}
- Trace.addTimelineAnnotation("query massaged");
// add extra parameters into olap context, like acceptPartial
Map<String, String> parameters = new HashMap<String, String>();