You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/20 07:33:44 UTC

[kylin] 01/02: KYLIN-3415 Remove external module

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit f77c7ba340189de3cdc4c761770a85bcdd1dd187
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Jun 19 18:05:08 2018 +0800

    KYLIN-3415 Remove external module
---
 core-common/pom.xml                                |  6 --
 .../org/apache/kylin/common/htrace/HtraceInit.java | 94 ----------------------
 .../kylin/storage/gtrecord/ScannerWorker.java      | 33 ++++----
 pom.xml                                            |  1 -
 .../kylin/query/enumerator/OLAPEnumerator.java     | 30 +++----
 .../apache/kylin/rest/service/QueryService.java    | 48 -----------
 6 files changed, 27 insertions(+), 185 deletions(-)

diff --git a/core-common/pom.xml b/core-common/pom.xml
index 47bdab9..8dca01f 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -67,12 +67,6 @@
             <artifactId>jsch</artifactId>
         </dependency>
 
-        <!-- Metric & Tracing -->
-        <dependency>
-            <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-external-htrace</artifactId>
-        </dependency>
-
         <!-- Env & Test -->
         <dependency>
             <groupId>junit</groupId>
diff --git a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java b/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
deleted file mode 100644
index c685947..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/htrace/HtraceInit.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.htrace;
-
-import java.io.File;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.HTraceConfiguration;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.SpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.LocalFileSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.StandardOutSpanReceiver;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.impl.ZipkinSpanReceiver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
-public class HtraceInit {
-    public static final Logger logger = LoggerFactory.getLogger(HtraceInit.class);
-
-    static {
-        try {
-            // init for HTrace
-            String fileName = System.getProperty("spanFile");
-
-            Collection<SpanReceiver> rcvrs = new HashSet<SpanReceiver>();
-
-            // writes spans to a file if one is provided to maven with
-            // -DspanFile="FILENAME", otherwise writes to standard out.
-            if (fileName != null) {
-                File f = new File(fileName);
-                File parent = f.getParentFile();
-                if (parent != null && !parent.exists() && !parent.mkdirs()) {
-                    throw new IllegalArgumentException("Couldn't create file: " + fileName);
-                }
-                HashMap<String, String> conf = new HashMap<String, String>();
-                conf.put("local-file-span-receiver.path", fileName);
-                LocalFileSpanReceiver receiver = new LocalFileSpanReceiver(HTraceConfiguration.fromMap(conf));
-                rcvrs.add(receiver);
-            } else {
-                rcvrs.add(new StandardOutSpanReceiver(HTraceConfiguration.EMPTY));
-            }
-
-            String hostKey = "zipkin.collector-hostname";
-            String host = System.getProperty(hostKey);
-            String portKey = "zipkin.collector-port";
-            String port = System.getProperty(portKey);
-
-            Map<String, String> confMap = Maps.newHashMap();
-            if (!StringUtils.isEmpty(host)) {
-                confMap.put(hostKey, host);
-                logger.info("{} is set to {}", hostKey, host);
-            }
-            if (!StringUtils.isEmpty(port)) {
-                confMap.put(portKey, port);
-                logger.info("{} is set to {}", portKey, port);
-            }
-
-            ZipkinSpanReceiver zipkinSpanReceiver = new ZipkinSpanReceiver(HTraceConfiguration.fromMap(confMap));
-            rcvrs.add(zipkinSpanReceiver);
-
-            for (SpanReceiver receiver : rcvrs) {
-                Trace.addReceiver(receiver);
-                logger.info("SpanReceiver {} added.", receiver);
-            }
-        } catch (Exception e) {
-            //
-            logger.error("Failed to init HTrace", e);
-        }
-    }
-
-    public static void init() { //only to trigger static block
-    }
-}
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
index def7e58..0a445fe 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -30,8 +30,6 @@ import org.apache.kylin.gridtable.GTScanRequest;
 import org.apache.kylin.gridtable.IGTScanner;
 import org.apache.kylin.gridtable.IGTStorage;
 import org.apache.kylin.metadata.model.ISegment;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
 import org.apache.kylin.storage.StorageContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,26 +46,23 @@ public class ScannerWorker {
         
         inputArgs = new Object[] { segment, cuboid, scanRequest, gtStorage, context };
         
-        try (TraceScope scope = Trace.startSpan("visit segment " + segment.getName())) {
-
-            if (scanRequest == null) {
-                logger.info("Segment {} will be skipped", segment);
-                internal = new EmptyGTScanner();
-                return;
-            }
+        if (scanRequest == null) {
+            logger.info("Segment {} will be skipped", segment);
+            internal = new EmptyGTScanner();
+            return;
+        }
 
-            final GTInfo info = scanRequest.getInfo();
+        final GTInfo info = scanRequest.getInfo();
 
-            try {
-                IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
-                        .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
-                        .newInstance(segment, cuboid, info, context); // default behavior
-                internal = rpc.getGTScanner(scanRequest);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
+        try {
+            IGTStorage rpc = (IGTStorage) Class.forName(gtStorage)
+                    .getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class)
+                    .newInstance(segment, cuboid, info, context); // default behavior
+            internal = rpc.getGTScanner(scanRequest);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
         }
-        
+
         checkNPE();
     }
 
diff --git a/pom.xml b/pom.xml
index 85b6b67..596f798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1163,7 +1163,6 @@
     </build>
 
     <modules>
-        <module>external</module>
         <module>atopcalcite</module>
         <module>core-common</module>
         <module>core-metadata</module>
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 721e00a..3f7beff 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -26,8 +26,6 @@ import org.apache.kylin.metadata.realization.SQLDigest;
 import org.apache.kylin.metadata.tuple.ITuple;
 import org.apache.kylin.metadata.tuple.ITupleIterator;
 import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
 import org.apache.kylin.storage.IStorageQuery;
 import org.apache.kylin.storage.StorageFactory;
 import org.slf4j.Logger;
@@ -103,25 +101,23 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
     }
 
     private ITupleIterator queryStorage() {
-        try (TraceScope scope = Trace.startSpan("query realization " + olapContext.realization.getCanonicalName())) {
 
-            logger.debug("query storage...");
-            // bind dynamic variables
-            olapContext.bindVariable(optiqContext);
+        logger.debug("query storage...");
+        // bind dynamic variables
+        olapContext.bindVariable(optiqContext);
 
-            olapContext.resetSQLDigest();
-            SQLDigest sqlDigest = olapContext.getSQLDigest();
+        olapContext.resetSQLDigest();
+        SQLDigest sqlDigest = olapContext.getSQLDigest();
 
-            // query storage engine
-            IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
-            ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
-                    olapContext.returnTupleInfo);
-            if (logger.isDebugEnabled()) {
-                logger.debug("return TupleIterator...");
-            }
-
-            return iterator;
+        // query storage engine
+        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
+        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
+                olapContext.returnTupleInfo);
+        if (logger.isDebugEnabled()) {
+            logger.debug("return TupleIterator...");
         }
+
+        return iterator;
     }
 
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 5f67aa1..b71f6ad 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -41,7 +41,6 @@ import org.apache.kylin.common.QueryContext;
 import org.apache.kylin.common.QueryContextFacade;
 import org.apache.kylin.common.debug.BackdoorToggles;
 import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
-import org.apache.kylin.common.htrace.HtraceInit;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.persistence.Serializer;
@@ -86,9 +85,6 @@ import org.apache.kylin.rest.util.AclEvaluate;
 import org.apache.kylin.rest.util.AclPermissionUtil;
 import org.apache.kylin.rest.util.QueryRequestLimits;
 import org.apache.kylin.rest.util.TableauInterceptor;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Sampler;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.Trace;
-import org.apache.kylin.shaded.htrace.org.apache.htrace.TraceScope;
 import org.apache.kylin.storage.hybrid.HybridInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -103,7 +99,6 @@ import javax.annotation.PostConstruct;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -412,14 +407,6 @@ public class QueryService extends BasicService {
 
         final QueryContext queryContext = QueryContextFacade.current();
 
-        TraceScope scope = null;
-        if (kylinConfig.isHtraceTracingEveryQuery() || BackdoorToggles.getHtraceEnabled()) {
-            logger.info("Current query is under tracing");
-            HtraceInit.init();
-            scope = Trace.startSpan("query life cycle for " + queryContext.getQueryId(), Sampler.ALWAYS);
-        }
-        String traceUrl = getTraceUrl(scope);
-
         try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId())) {
             SQLResponse sqlResponse = null;
             String sql = sqlRequest.getSql();
@@ -446,7 +433,6 @@ public class QueryService extends BasicService {
 
             if (sqlResponse == null && isQueryCacheEnabled) {
                 sqlResponse = searchQueryInCache(sqlRequest);
-                Trace.addTimelineAnnotation("query cache searched");
             }
 
             // real execution if required
@@ -454,8 +440,6 @@ public class QueryService extends BasicService {
                 try (QueryRequestLimits limit = new QueryRequestLimits(sqlRequest.getProject())) {
                     sqlResponse = queryAndUpdateCache(sqlRequest, isQueryCacheEnabled);
                 }
-            } else {
-                Trace.addTimelineAnnotation("response without real execution");
             }
 
             // check authorization before return, since the response may come from cache
@@ -463,7 +447,6 @@ public class QueryService extends BasicService {
                 checkQueryAuth(sqlResponse, project);
 
             sqlResponse.setDuration(queryContext.getAccumulatedMillis());
-            sqlResponse.setTraceUrl(traceUrl);
             logQuery(queryContext.getQueryId(), sqlRequest, sqlResponse);
             try {
                 recordMetric(sqlRequest, sqlResponse);
@@ -478,9 +461,6 @@ public class QueryService extends BasicService {
         } finally {
             BackdoorToggles.cleanToggles();
             QueryContextFacade.resetCurrent();
-            if (scope != null) {
-                scope.close();
-            }
         }
     }
 
@@ -494,10 +474,8 @@ public class QueryService extends BasicService {
             final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
             if (isSelect) {
                 sqlResponse = query(sqlRequest, queryContext.getQueryId());
-                Trace.addTimelineAnnotation("query almost done");
             } else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled()) {
                 sqlResponse = update(sqlRequest);
-                Trace.addTimelineAnnotation("update query almost done");
             } else {
                 logger.debug("Directly return exception as the sql is unsupported, and query pushdown is disabled");
                 throw new BadRequestException(msg.getNOT_SUPPORTED_SQL());
@@ -528,7 +506,6 @@ public class QueryService extends BasicService {
                             kylinConfig.getLargeQueryThreshold())) {
                 cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest.getCacheKey(), sqlResponse));
             }
-            Trace.addTimelineAnnotation("response from execution");
 
         } catch (Throwable e) { // calcite may throw AssertError
             queryContext.stop(e);
@@ -546,7 +523,6 @@ public class QueryService extends BasicService {
                 Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
                 exceptionCache.put(new Element(sqlRequest.getCacheKey(), sqlResponse));
             }
-            Trace.addTimelineAnnotation("error response");
         }
         return sqlResponse;
     }
@@ -561,29 +537,6 @@ public class QueryService extends BasicService {
         QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse);
     }
 
-    private String getTraceUrl(TraceScope scope) {
-        if (scope == null) {
-            return null;
-        }
-
-        String hostname = System.getProperty("zipkin.collector-hostname");
-        if (StringUtils.isEmpty(hostname)) {
-            try {
-                hostname = InetAddress.getLocalHost().getHostName();
-            } catch (UnknownHostException e) {
-                logger.debug("failed to get trace url due to " + e.getMessage());
-                return null;
-            }
-        }
-
-        String port = System.getProperty("zipkin.web-ui-port");
-        if (StringUtils.isEmpty(port)) {
-            port = "9411";
-        }
-
-        return "http://" + hostname + ":" + port + "/zipkin/traces/" + Long.toHexString(scope.getSpan().getTraceId());
-    }
-
     private String getUserName() {
         String username = SecurityContextHolder.getContext().getAuthentication().getName();
         if (StringUtils.isEmpty(username)) {
@@ -648,7 +601,6 @@ public class QueryService extends BasicService {
                 //CAUTION: should not change sqlRequest content!
                 //sqlRequest.setSql(correctedSql);
             }
-            Trace.addTimelineAnnotation("query massaged");
 
             // add extra parameters into olap context, like acceptPartial
             Map<String, String> parameters = new HashMap<String, String>();