You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/20 08:53:22 UTC
[kylin] branch master updated: KYLIN-3415 Remove external module
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 0f6273e KYLIN-3415 Remove external module
0f6273e is described below
commit 0f6273e3ad0003082ed2585f1bc0339262a18350
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Jun 19 18:05:08 2018 +0800
KYLIN-3415 Remove external module
---
core-common/pom.xml | 6 --
.../org/apache/kylin/common/htrace/HtraceInit.java | 94 -------------------
.../kylin/storage/gtrecord/ScannerWorker.java | 33 +++----
external/htrace/pom.xml | 102 ---------------------
external/pom.xml | 70 --------------
pom.xml | 6 --
.../kylin/query/enumerator/OLAPEnumerator.java | 30 +++---
.../apache/kylin/rest/service/QueryService.java | 48 ----------
8 files changed, 27 insertions(+), 362 deletions(-)
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 142a853..9151248 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -67,12 +67,6 @@
<artifactId>jsch</artifactId>
</dependency>
- <!-- Metric & Tracing -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-external-htrace</artifactId>
- </dependency>
-
<!-- Env & Test -->
<dependency>
<groupId>junit</groupId>
diff --git a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java b/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
deleted file mode 100644
index c685947..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.htrace;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.HTraceConfiguration;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.SpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.LocalFileSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.StandardOutSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.ZipkinSpanReceiver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
-public class HtraceInit {
- public static final Logger logger = LoggerFactory.getLogger(HtraceInit.class);
-
- static {
- try {
- // init for HTrace
- String fileName = System.getProperty("spanFile");
-
- Collection<SpanReceiver> rcvrs = new HashSet<SpanReceiver>();
-
- // writes spans to a file if one is provided to maven with
- // -DspanFile="FILENAME", otherwise writes to standard out.
- if (fileName != null) {
- File f = new File(fileName);
- File parent = f.getParentFile();
- if (parent != null && !parent.exists() && !parent.mkdirs()) {
- throw new IllegalArgumentException("Couldn't create file: " + fileName);
- }
- HashMap<String, String> conf = new HashMap<String, String>();
- conf.put("local-file-span-receiver.path", fileName);
- LocalFileSpanReceiver receiver = new LocalFileSpanReceiver(HTraceConfiguration.fromMap(conf));
- rcvrs.add(receiver);
- } else {
- rcvrs.add(new StandardOutSpanReceiver(HTraceConfiguration.EMPTY));
- }
-
- String hostKey = "zipkin.collector-hostname";
- String host = System.getProperty(hostKey);
- String portKey = "zipkin.collector-port";
- String port = System.getProperty(portKey);
-
- Map<String, String> confMap = Maps.newHashMap();
- if (!StringUtils.isEmpty(host)) {
- confMap.put(hostKey, host);
- logger.info("{} is set to {}", hostKey, host);
- }
- if (!StringUtils.isEmpty(port)) {
- confMap.put(portKey, port);
- logger.info("{} is set to {}", portKey, port);
- }
-
- ZipkinSpanReceiver zipkinSpanReceiver = new ZipkinSpanReceiver(HTraceConfiguration.fromMap(confMap));
- rcvrs.add(zipkinSpanReceiver);
-
- for (SpanReceiver receiver : rcvrs) {
- Trace.addReceiver(receiver);
- logger.info("SpanReceiver {} added.", receiver);
- }
- } catch (Exception e) {
- //
- logger.error("Failed to init HTrace", e);
- }
- }
-
- public static void init() { //only to trigger static block
- }
-}
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
index def7e58..0a445fe 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -30,8 +30,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStorage;
import org.apache.kylin.metadata.model.ISegment;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,26 +46,23 @@ public class ScannerWorker {
inputArgs = new Object[] { segment, cuboid, scanRequest, gtStorage, context };
- try (TraceScope scope = Trace.startSpan("visit segment " + segment.getName())) {
-
- if (scanRequest == null) {
- logger.info("Segment {} will be skipped", segment);
- internal = new EmptyGTScanner();
- return;
- }
+ if (scanRequest == null) {
+ logger.info("Segment {} will be skipped", segment);
+ internal = new EmptyGTScanner();
+ return;
+ }
- final GTInfo info = scanRequest.getInfo();
+ final GTInfo info = scanRequest.getInfo();
- try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
- .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
- .newInstance(segment, cuboid, info, context); // default behavior
- internal = rpc.getGTScanner(scanRequest);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ try {
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
+ .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
+ .newInstance(segment, cuboid, info, context); // default behavior
+ internal = rpc.getGTScanner(scanRequest);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
-
+
checkNPE();
}
diff --git a/external/htrace/pom.xml b/external/htrace/pom.xml
deleted file mode 100644
index cfadfce..0000000
--- a/external/htrace/pom.xml
+++ /dev/null
@@ -1,102 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-external-htrace</artifactId>
- <packaging>jar</packaging>
- <name>Apache Kylin - External HTrace</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>2.4.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <shadeBase>org.apache.kylin.shaded.htrace</shadeBase>
- <shaded.htrace.version>3.1.0-incubating</shaded.htrace.version>
- <checkstyle.skip>true</checkstyle.skip>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- <version>${shaded.htrace.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-zipkin</artifactId>
- <version>${shaded.htrace.version}</version>
- </dependency>
- </dependencies>
- <!--overwrite parent, need to upgrade this when upgrade grpc-->
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <excludes>
- <exclude>org.slf4j:*</exclude>
- </excludes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>com.twitter.zipkin.gen</pattern>
- <shadedPattern>${shadeBase}.com.twitter.zipkin.gen
- </shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache</pattern>
- <shadedPattern>${shadeBase}.org.apache
- </shadedPattern>
- </relocation>
- </relocations>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
-</project>
diff --git a/external/pom.xml b/external/pom.xml
deleted file mode 100644
index 8a038ba..0000000
--- a/external/pom.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-external</artifactId>
- <packaging>pom</packaging>
- <name>Apache Kylin - External</name>
-
- <parent>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin</artifactId>
- <version>2.4.0-SNAPSHOT</version>
- </parent>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <!--overwrite parent, need to upgrade this when upgrade grpc-->
- <build>
- <plugins>
- <plugin>
- <artifactId>exec-maven-plugin</artifactId>
- <groupId>org.codehaus.mojo</groupId>
- <version>1.6.0</version>
- <executions>
- <execution>
- <id>install_htrace</id>
- <phase>compile</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>mvn</executable>
- <arguments>
- <argument>-f</argument>
- <argument>
- external/htrace/pom.xml
- </argument>
- <argument>clean</argument>
- <argument>install</argument>
- <argument>-DskipTests</argument>
- </arguments>
- <workingDirectory>${project.basedir}/..
- </workingDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/pom.xml b/pom.xml
index bfcbcb2..fe424f8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -473,11 +473,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-external-htrace</artifactId>
- <version>2.4.0-SNAPSHOT</version>
- </dependency>
- <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
@@ -1175,7 +1170,6 @@
</build>
<modules>
- <module>external</module>
<module>atopcalcite</module>
<module>core-common</module>
<module>core-metadata</module>
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 721e00a..3f7beff 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -26,8 +26,6 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.StorageFactory;
import org.slf4j.Logger;
@@ -103,25 +101,23 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
}
private ITupleIterator queryStorage() {
- try (TraceScope scope = Trace.startSpan("query realization " + olapContext.realization.getCanonicalName())) {
- logger.debug("query storage...");
- // bind dynamic variables
- olapContext.bindVariable(optiqContext);
+ logger.debug("query storage...");
+ // bind dynamic variables
+ olapContext.bindVariable(optiqContext);
- olapContext.resetSQLDigest();
- SQLDigest sqlDigest = olapContext.getSQLDigest();
+ olapContext.resetSQLDigest();
+ SQLDigest sqlDigest = olapContext.getSQLDigest();
- // query storage engine
- IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
- ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
- olapContext.returnTupleInfo);
- if (logger.isDebugEnabled()) {
- logger.debug("return TupleIterator...");
- }
-
- return iterator;
+ // query storage engine
+ IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
+ ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
+ olapContext.returnTupleInfo);
+ if (logger.isDebugEnabled()) {
+ logger.debug("return TupleIterator...");
}
+
+ return iterator;
}
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 375b069..4e3fe07 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -23,7 +23,6 @@ import static org.apache.kylin.common.util.CheckUtil.checkCondition;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
@@ -61,7 +60,6 @@ import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryContextFacade;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
-import org.apache.kylin.common.htrace.HtraceInit;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
@@ -103,9 +101,6 @@ import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
import org.apache.kylin.rest.util.TableauInterceptor;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Sampler;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -352,14 +347,6 @@ public class QueryService extends BasicService {
final QueryContext queryContext = QueryContextFacade.current();
- TraceScope scope = null;
- if (kylinConfig.isHtraceTracingEveryQuery() || BackdoorToggles.getHtraceEnabled()) {
- logger.info("Current query is under tracing");
- HtraceInit.init();
- scope = Trace.startSpan("query life cycle for " + queryContext.getQueryId(), Sampler.ALWAYS);
- }
- String traceUrl = getTraceUrl(scope);
-
try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId())) {
SQLResponse sqlResponse = null;
String sql = sqlRequest.getSql();
@@ -386,7 +373,6 @@ public class QueryService extends BasicService {
if (sqlResponse == null && isQueryCacheEnabled) {
sqlResponse = searchQueryInCache(sqlRequest);
- Trace.addTimelineAnnotation("query cache searched");
}
// real execution if required
@@ -394,12 +380,9 @@ public class QueryService extends BasicService {
try (QueryRequestLimits limit = new QueryRequestLimits(sqlRequest.getProject())) {
sqlResponse = queryAndUpdateCache(sqlRequest, isQueryCacheEnabled);
}
- } else {
- Trace.addTimelineAnnotation("response without real execution");
}
sqlResponse.setDuration(queryContext.getAccumulatedMillis());
- sqlResponse.setTraceUrl(traceUrl);
logQuery(queryContext.getQueryId(), sqlRequest, sqlResponse);
try {
recordMetric(sqlRequest, sqlResponse);
@@ -414,9 +397,6 @@ public class QueryService extends BasicService {
} finally {
BackdoorToggles.cleanToggles();
QueryContextFacade.resetCurrent();
- if (scope != null) {
- scope.close();
- }
}
}
@@ -430,10 +410,8 @@ public class QueryService extends BasicService {
final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
if (isSelect) {
sqlResponse = query(sqlRequest, queryContext.getQueryId());
- Trace.addTimelineAnnotation("query almost done");
} else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled()) {
sqlResponse = update(sqlRequest);
- Trace.addTimelineAnnotation("update query almost done");
} else {
logger.debug("Directly return exception as the sql is unsupported, and query pushdown is disabled");
throw new BadRequestException(msg.getNOT_SUPPORTED_SQL());
@@ -464,7 +442,6 @@ public class QueryService extends BasicService {
kylinConfig.getLargeQueryThreshold())) {
cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
- Trace.addTimelineAnnotation("response from execution");
} catch (Throwable e) { // calcite may throw AssertError
queryContext.stop(e);
@@ -482,7 +459,6 @@ public class QueryService extends BasicService {
Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
}
- Trace.addTimelineAnnotation("error response");
}
return sqlResponse;
}
@@ -497,29 +473,6 @@ public class QueryService extends BasicService {
QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
}
- private String getTraceUrl(TraceScope scope) {
- if (scope == null) {
- return null;
- }
-
- String hostname = System.getProperty("zipkin.collector-hostname");
- if (StringUtils.isEmpty(hostname)) {
- try {
- hostname = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- logger.debug("failed to get trace url due to " + e.getMessage());
- return null;
- }
- }
-
- String port = System.getProperty("zipkin.web-ui-port");
- if (StringUtils.isEmpty(port)) {
- port = "9411";
- }
-
- return "http://" + hostname + ":" + port + "/zipkin/traces/" + Long.toHexString(scope.getSpan().getTraceId());
- }
-
private String getUserName() {
String username = SecurityContextHolder.getContext().getAuthentication().getName();
if (StringUtils.isEmpty(username)) {
@@ -578,7 +531,6 @@ public class QueryService extends BasicService {
//CAUTION: should not change sqlRequest content!
//sqlRequest.setSql(correctedSql);
}
- Trace.addTimelineAnnotation("query massaged");
// add extra parameters into olap context, like acceptPartial
Map<String, String> parameters = new HashMap<String, String>();