You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/20 07:33:43 UTC
[kylin] branch 2.3.x updated (46b876f -> e177eed)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 46b876f KYLIN-3400 wipeCache and createCubeDesc make deadlock
new f77c7ba KYLIN-3415 Remove external module
new e177eed Revert "KYLIN-3094 Upgrade zookeeper version from 3.4.8 to 3.4.12"
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
core-common/pom.xml | 6 --
.../org/apache/kylin/common/htrace/HtraceInit.java | 94 ----------------------
.../kylin/storage/gtrecord/ScannerWorker.java | 33 ++++----
pom.xml | 5 +-
.../kylin/query/enumerator/OLAPEnumerator.java | 30 +++----
.../apache/kylin/rest/service/QueryService.java | 48 -----------
6 files changed, 29 insertions(+), 187 deletions(-)
delete mode 100644 core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
[kylin] 01/02: KYLIN-3415 Remove external module
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f77c7ba340189de3cdc4c761770a85bcdd1dd187
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Jun 19 18:05:08 2018 +0800
KYLIN-3415 Remove external module
---
core-common/pom.xml | 6 --
.../org/apache/kylin/common/htrace/HtraceInit.java | 94 ----------------------
.../kylin/storage/gtrecord/ScannerWorker.java | 33 ++++----
pom.xml | 1 -
.../kylin/query/enumerator/OLAPEnumerator.java | 30 +++----
.../apache/kylin/rest/service/QueryService.java | 48 -----------
6 files changed, 27 insertions(+), 185 deletions(-)
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 47bdab9..8dca01f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -67,12 +67,6 @@
<artifactId>jsch</artifactId>
</dependency>
- <!-- Metric & Tracing -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-external-htrace</artifactId>
- </dependency>
-
<!-- Env & Test -->
<dependency>
<groupId>junit</groupId>
diff --git a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java b/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
deleted file mode 100644
index c685947..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.htrace;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.HTraceConfiguration;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.SpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.LocalFileSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.StandardOutSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.ZipkinSpanReceiver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
-public class HtraceInit {
- public static final Logger logger = LoggerFactory.getLogger(HtraceInit.class);
-
- static {
- try {
- // init for HTrace
- String fileName = System.getProperty("spanFile");
-
- Collection<SpanReceiver> rcvrs = new HashSet<SpanReceiver>();
-
- // writes spans to a file if one is provided to maven with
- // -DspanFile="FILENAME", otherwise writes to standard out.
- if (fileName != null) {
- File f = new File(fileName);
- File parent = f.getParentFile();
- if (parent != null && !parent.exists() && !parent.mkdirs()) {
- throw new IllegalArgumentException("Couldn't create file: " + fileName);
- }
- HashMap<String, String> conf = new HashMap<String, String>();
- conf.put("local-file-span-receiver.path", fileName);
- LocalFileSpanReceiver receiver = new LocalFileSpanReceiver(HTraceConfiguration.fromMap(conf));
- rcvrs.add(receiver);
- } else {
- rcvrs.add(new StandardOutSpanReceiver(HTraceConfiguration.EMPTY));
- }
-
- String hostKey = "zipkin.collector-hostname";
- String host = System.getProperty(hostKey);
- String portKey = "zipkin.collector-port";
- String port = System.getProperty(portKey);
-
- Map<String, String> confMap = Maps.newHashMap();
- if (!StringUtils.isEmpty(host)) {
- confMap.put(hostKey, host);
- logger.info("{} is set to {}", hostKey, host);
- }
- if (!StringUtils.isEmpty(port)) {
- confMap.put(portKey, port);
- logger.info("{} is set to {}", portKey, port);
- }
-
- ZipkinSpanReceiver zipkinSpanReceiver = new ZipkinSpanReceiver(HTraceConfiguration.fromMap(confMap));
- rcvrs.add(zipkinSpanReceiver);
-
- for (SpanReceiver receiver : rcvrs) {
- Trace.addReceiver(receiver);
- logger.info("SpanReceiver {} added.", receiver);
- }
- } catch (Exception e) {
- //
- logger.error("Failed to init HTrace", e);
- }
- }
-
- public static void init() { //only to trigger static block
- }
-}
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
index def7e58..0a445fe 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -30,8 +30,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStorage;
import org.apache.kylin.metadata.model.ISegment;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,26 +46,23 @@ public class ScannerWorker {
inputArgs = new Object[] { segment, cuboid, scanRequest, gtStorage, context };
- try (TraceScope scope = Trace.startSpan("visit segment " + segment.getName())) {
-
- if (scanRequest == null) {
- logger.info("Segment {} will be skipped", segment);
- internal = new EmptyGTScanner();
- return;
- }
+ if (scanRequest == null) {
+ logger.info("Segment {} will be skipped", segment);
+ internal = new EmptyGTScanner();
+ return;
+ }
- final GTInfo info = scanRequest.getInfo();
+ final GTInfo info = scanRequest.getInfo();
- try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
- .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
- .newInstance(segment, cuboid, info, context); // default behavior
- internal = rpc.getGTScanner(scanRequest);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ try {
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
+ .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
+ .newInstance(segment, cuboid, info, context); // default behavior
+ internal = rpc.getGTScanner(scanRequest);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
-
+
checkNPE();
}
diff --git a/pom.xml b/pom.xml
index 85b6b67..596f798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1163,7 +1163,6 @@
</build>
<modules>
- <module>external</module>
<module>atopcalcite</module>
<module>core-common</module>
<module>core-metadata</module>
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 721e00a..3f7beff 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -26,8 +26,6 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.StorageFactory;
import org.slf4j.Logger;
@@ -103,25 +101,23 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
}
private ITupleIterator queryStorage() {
- try (TraceScope scope = Trace.startSpan("query realization " + olapContext.realization.getCanonicalName())) {
- logger.debug("query storage...");
- // bind dynamic variables
- olapContext.bindVariable(optiqContext);
+ logger.debug("query storage...");
+ // bind dynamic variables
+ olapContext.bindVariable(optiqContext);
- olapContext.resetSQLDigest();
- SQLDigest sqlDigest = olapContext.getSQLDigest();
+ olapContext.resetSQLDigest();
+ SQLDigest sqlDigest = olapContext.getSQLDigest();
- // query storage engine
- IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
- ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
- olapContext.returnTupleInfo);
- if (logger.isDebugEnabled()) {
- logger.debug("return TupleIterator...");
- }
-
- return iterator;
+ // query storage engine
+ IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
+ ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
+ olapContext.returnTupleInfo);
+ if (logger.isDebugEnabled()) {
+ logger.debug("return TupleIterator...");
}
+
+ return iterator;
}
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 5f67aa1..b71f6ad 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -41,7 +41,6 @@ import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryContextFacade;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
-import org.apache.kylin.common.htrace.HtraceInit;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
@@ -86,9 +85,6 @@ import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
import org.apache.kylin.rest.util.TableauInterceptor;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Sampler;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -103,7 +99,6 @@ import javax.annotation.PostConstruct;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -412,14 +407,6 @@ public class QueryService extends BasicService {
final QueryContext queryContext = QueryContextFacade.current();
- TraceScope scope = null;
- if (kylinConfig.isHtraceTracingEveryQuery() || BackdoorToggles.getHtraceEnabled()) {
- logger.info("Current query is under tracing");
- HtraceInit.init();
- scope = Trace.startSpan("query life cycle for " + queryContext.getQueryId(), Sampler.ALWAYS);
- }
- String traceUrl = getTraceUrl(scope);
-
try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId())) {
SQLResponse sqlResponse = null;
String sql = sqlRequest.getSql();
@@ -446,7 +433,6 @@ public class QueryService extends BasicService {
if (sqlResponse == null && isQueryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
- Trace.addTimelineAnnotation("query cache searched");
}
// real execution if required
@@ -454,8 +440,6 @@ public class QueryService extends BasicService {
try (QueryRequestLimits limit = new QueryRequestLimits(sqlRequest.getProject())) {
sqlResponse = queryAndUpdateCache(sqlRequest, isQueryCacheEnabled);
}
- } else {
- Trace.addTimelineAnnotation("response without real execution");
}
// check authorization before return, since the response may come from cache
@@ -463,7 +447,6 @@ public class QueryService extends BasicService {
checkQueryAuth(sqlResponse, project);
sqlResponse.setDuration(queryContext.getAccumulatedMillis());
- sqlResponse.setTraceUrl(traceUrl);
logQuery(queryContext.getQueryId(), sqlRequest, sqlResponse);
try {
recordMetric(sqlRequest, sqlResponse);
@@ -478,9 +461,6 @@ public class QueryService extends BasicService {
} finally {
BackdoorToggles.cleanToggles();
QueryContextFacade.resetCurrent();
- if (scope != null) {
- scope.close();
- }
}
}
@@ -494,10 +474,8 @@ public class QueryService extends BasicService {
final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
if (isSelect) {
sqlResponse = query(sqlRequest, queryContext.getQueryId());
- Trace.addTimelineAnnotation("query almost done");
} else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled()) {
sqlResponse = update(sqlRequest);
- Trace.addTimelineAnnotation("update query almost done");
} else {
logger.debug("Directly return exception as the sql is unsupported, and query pushdown is disabled");
throw new BadRequestException(msg.getNOT_SUPPORTED_SQL());
@@ -528,7 +506,6 @@ public class QueryService extends BasicService {
kylinConfig.getLargeQueryThreshold())) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
- Trace.addTimelineAnnotation("response from execution");
} catch (Throwable e) { // calcite may throw AssertError
queryContext.stop(e);
@@ -546,7 +523,6 @@ public class QueryService extends BasicService {
Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
- Trace.addTimelineAnnotation("error response");
}
return sqlResponse;
}
@@ -561,29 +537,6 @@ public class QueryService extends BasicService {
QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
}
- private String getTraceUrl(TraceScope scope) {
- if (scope == null) {
- return null;
- }
-
- String hostname = System.getProperty("zipkin.collector-hostname");
- if (StringUtils.isEmpty(hostname)) {
- try {
- hostname = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- logger.debug("failed to get trace url due to " + e.getMessage());
- return null;
- }
- }
-
- String port = System.getProperty("zipkin.web-ui-port");
- if (StringUtils.isEmpty(port)) {
- port = "9411";
- }
-
- return "http://" + hostname + ":" + port + "/zipkin/traces/" + Long.toHexString(scope.getSpan().getTraceId());
- }
-
private String getUserName() {
String username = SecurityContextHolder.getContext().getAuthentication().getName();
if (StringUtils.isEmpty(username)) {
@@ -648,7 +601,6 @@ public class QueryService extends BasicService {
//CAUTION: should not change sqlRequest content!
//sqlRequest.setSql(correctedSql);
}
- Trace.addTimelineAnnotation("query massaged");
// add extra parameters into olap context, like acceptPartial
Map<String, String> parameters = new HashMap<String, String>();
[kylin] 02/02: Revert "KYLIN-3094 Upgrade zookeeper version from
3.4.8 to 3.4.12"
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e177eed39a0caedf7e44c5c35b7ce1ac49765cc5
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Jun 20 15:16:37 2018 +0800
Revert "KYLIN-3094 Upgrade zookeeper version from 3.4.8 to 3.4.12"
This reverts commit 1e29f8aaa316a8d6000e73530b2310d73f9093b9.
---
pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pom.xml b/pom.xml
index 596f798..d4afc13 100644
--- a/pom.xml
+++ b/pom.xml
@@ -69,8 +69,8 @@
<avatica.version>1.10.0</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
- <zookeeper.version>3.4.12</zookeeper.version>
- <curator.version>2.12.0</curator.version>
+ <zookeeper.version>3.4.8</zookeeper.version>
+ <curator.version>2.7.1</curator.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
<jsch.version>0.1.53</jsch.version>