You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by zs...@apache.org on 2016/10/07 17:31:45 UTC
spark git commit: [SPARK-17707][WEBUI] Web UI prevents spark-submit
application to be finished
Repository: spark
Updated Branches:
refs/heads/master dd16b52cf -> cff560755
[SPARK-17707][WEBUI] Web UI prevents spark-submit application to be finished
## What changes were proposed in this pull request?
This expands calls to Jetty's simple `ServerConnector` constructor to explicitly specify a `ScheduledExecutorScheduler` that makes daemon threads. It should otherwise result in exactly the same configuration, because the other args are copied from the constructor that is currently called.
(I'm not sure we should change the Hive Thriftserver impl, but I did anyway.)
This also adds `sc.stop()` to the quick start guide example.
## How was this patch tested?
Existing tests; _pending_ at least manual verification of the fix.
Author: Sean Owen <so...@cloudera.com>
Closes #15381 from srowen/SPARK-17707.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cff56075
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cff56075
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cff56075
Branch: refs/heads/master
Commit: cff560755244dd4ccb998e0c56e81d2620cd4cff
Parents: dd16b52
Author: Sean Owen <so...@cloudera.com>
Authored: Fri Oct 7 10:31:41 2016 -0700
Committer: Shixiong Zhu <sh...@databricks.com>
Committed: Fri Oct 7 10:31:41 2016 -0700
----------------------------------------------------------------------
.../deploy/rest/RestSubmissionServer.scala | 14 ++++++++++---
.../scala/org/apache/spark/ui/JettyUtils.scala | 14 ++++++++++---
docs/quick-start.md | 7 ++++++-
.../cli/thrift/ThriftHttpCLIService.java | 22 ++++++++++++++++++--
4 files changed, 48 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/cff56075/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
index fa55d47..b30c980 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
@@ -22,9 +22,9 @@ import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse}
import scala.io.Source
import com.fasterxml.jackson.core.JsonProcessingException
-import org.eclipse.jetty.server.{Server, ServerConnector}
+import org.eclipse.jetty.server.{HttpConnectionFactory, Server, ServerConnector}
import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
-import org.eclipse.jetty.util.thread.QueuedThreadPool
+import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler}
import org.json4s._
import org.json4s.jackson.JsonMethods._
@@ -83,7 +83,15 @@ private[spark] abstract class RestSubmissionServer(
threadPool.setDaemon(true)
val server = new Server(threadPool)
- val connector = new ServerConnector(server)
+ val connector = new ServerConnector(
+ server,
+ null,
+ // Call this full constructor to set this, which forces daemon threads:
+ new ScheduledExecutorScheduler("RestSubmissionServer-JettyScheduler", true),
+ null,
+ -1,
+ -1,
+ new HttpConnectionFactory())
connector.setHost(host)
connector.setPort(startPort)
server.addConnector(connector)
http://git-wip-us.apache.org/repos/asf/spark/blob/cff56075/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 24f3f75..35c3c8d 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -27,12 +27,12 @@ import scala.xml.Node
import org.eclipse.jetty.client.api.Response
import org.eclipse.jetty.proxy.ProxyServlet
-import org.eclipse.jetty.server.{Request, Server, ServerConnector}
+import org.eclipse.jetty.server.{HttpConnectionFactory, Request, Server, ServerConnector}
import org.eclipse.jetty.server.handler._
import org.eclipse.jetty.servlet._
import org.eclipse.jetty.servlets.gzip.GzipHandler
import org.eclipse.jetty.util.component.LifeCycle
-import org.eclipse.jetty.util.thread.QueuedThreadPool
+import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler}
import org.json4s.JValue
import org.json4s.jackson.JsonMethods.{pretty, render}
@@ -294,7 +294,15 @@ private[spark] object JettyUtils extends Logging {
val server = new Server(pool)
val connectors = new ArrayBuffer[ServerConnector]
// Create a connector on port currentPort to listen for HTTP requests
- val httpConnector = new ServerConnector(server)
+ val httpConnector = new ServerConnector(
+ server,
+ null,
+ // Call this full constructor to set this, which forces daemon threads:
+ new ScheduledExecutorScheduler(s"$serverName-JettyScheduler", true),
+ null,
+ -1,
+ -1,
+ new HttpConnectionFactory())
httpConnector.setPort(currentPort)
connectors += httpConnector
http://git-wip-us.apache.org/repos/asf/spark/blob/cff56075/docs/quick-start.md
----------------------------------------------------------------------
diff --git a/docs/quick-start.md b/docs/quick-start.md
index 2eab8d1..cb9a378 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -240,7 +240,8 @@ object SimpleApp {
val logData = sc.textFile(logFile, 2).cache()
val numAs = logData.filter(line => line.contains("a")).count()
val numBs = logData.filter(line => line.contains("b")).count()
- println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
+ println(s"Lines with a: $numAs, Lines with b: $numBs")
+ sc.stop()
}
}
{% endhighlight %}
@@ -328,6 +329,8 @@ public class SimpleApp {
}).count();
System.out.println("Lines with a: " + numAs + ", lines with b: " + numBs);
+
+ sc.stop()
}
}
{% endhighlight %}
@@ -407,6 +410,8 @@ numAs = logData.filter(lambda s: 'a' in s).count()
numBs = logData.filter(lambda s: 'b' in s).count()
print("Lines with a: %i, lines with b: %i" % (numAs, numBs))
+
+sc.stop()
{% endhighlight %}
http://git-wip-us.apache.org/repos/asf/spark/blob/cff56075/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 37e4845..341a7fd 100644
--- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -37,11 +37,15 @@ import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServlet;
+import org.eclipse.jetty.server.AbstractConnectionFactory;
+import org.eclipse.jetty.server.ConnectionFactory;
+import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
+import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
public class ThriftHttpCLIService extends ThriftCLIService {
@@ -70,7 +74,8 @@ public class ThriftHttpCLIService extends ThriftCLIService {
httpServer = new org.eclipse.jetty.server.Server(threadPool);
// Connector configs
- ServerConnector connector = new ServerConnector(httpServer);
+
+ ConnectionFactory[] connectionFactories;
boolean useSsl = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
String schemeName = useSsl ? "https" : "http";
// Change connector if SSL is used
@@ -90,8 +95,21 @@ public class ThriftHttpCLIService extends ThriftCLIService {
Arrays.toString(sslContextFactory.getExcludeProtocols()));
sslContextFactory.setKeyStorePath(keyStorePath);
sslContextFactory.setKeyStorePassword(keyStorePassword);
- connector = new ServerConnector(httpServer, sslContextFactory);
+ connectionFactories = AbstractConnectionFactory.getFactories(
+ sslContextFactory, new HttpConnectionFactory());
+ } else {
+ connectionFactories = new ConnectionFactory[] { new HttpConnectionFactory() };
}
+ ServerConnector connector = new ServerConnector(
+ httpServer,
+ null,
+ // Call this full constructor to set this, which forces daemon threads:
+ new ScheduledExecutorScheduler("HiveServer2-HttpHandler-JettyScheduler", true),
+ null,
+ -1,
+ -1,
+ connectionFactories);
+
connector.setPort(portNum);
// Linux:yes, Windows:no
connector.setReuseAddress(!Shell.WINDOWS);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org