You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2022/07/13 06:50:14 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #3052] Support to get the real client ip address for http connection when using VIP as kyuubi server load balancer
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a3973a0b3 [KYUUBI #3052] Support to get the real client ip address for http connection when using VIP as kyuubi server load balancer
a3973a0b3 is described below
commit a3973a0b3d483a0514d621353c6c6027ad04c6d1
Author: Fei Wang <fw...@ebay.com>
AuthorDate: Wed Jul 13 14:50:06 2022 +0800
[KYUUBI #3052] Support to get the real client ip address for http connection when using VIP as kyuubi server load balancer
### _Why are the changes needed?_
To close #3052
If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP herader that will be added to the request for other devices to use.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3053 from turboFei/http_real_ip.
Closes #3052
b375d02a [Fei Wang] add ut
23ef1c9c [Fei Wang] refactor
907c61c0 [Fei Wang] also take affect for http
a958c50f [Fei Wang] add conf
Authored-by: Fei Wang <fw...@ebay.com>
Signed-off-by: Fei Wang <fw...@ebay.com>
---
docs/deployment/settings.md | 1 +
.../org/apache/kyuubi/config/KyuubiConf.scala | 11 +++++++++
.../kyuubi/server/http/ThriftHttpServlet.scala | 4 +++-
.../http/authentication/AuthenticationFilter.scala | 5 +++--
.../server/api/v1/BatchesResourceSuite.scala | 26 ++++++++++++++++++++++
5 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 4d8d9fa6c..941d2bf9f 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -273,6 +273,7 @@ kyuubi.frontend.mysql.min.worker.threads|9|Minimum number of threads in the comm
kyuubi.frontend.mysql.netty.worker.threads|<undefined>|Number of thread in the netty worker event loop of MySQL frontend service. Use min(cpu_cores, 8) in default.|int|1.4.0
kyuubi.frontend.mysql.worker.keepalive.time|PT1M|Time(ms) that an idle async thread of the command execution thread pool will wait for a new task to arrive before terminating in MySQL frontend service|duration|1.4.0
kyuubi.frontend.protocols|THRIFT_BINARY|A comma separated list for all frontend protocols <ul> <li>THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.</li> <li>THRIFT_HTTP - HiveServer2 compatible thrift http protocol.</li> <li>REST - Kyuubi defined REST API(experimental).</li> <li>MYSQL - MySQL compatible text protocol(experimental).</li> </ul>|seq|1.4.0
+kyuubi.frontend.proxy.http.client.ip.header|X-Real-IP|The http header to record the real client ip address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP herader that will be added to the request for other devices to use.|string|1.6.0
kyuubi.frontend.rest.bind.host|<undefined>|Hostname or IP of the machine on which to run the REST frontend service.|string|1.4.0
kyuubi.frontend.rest.bind.port|10099|Port of the machine on which to run the REST frontend service.|int|1.4.0
kyuubi.frontend.thrift.backoff.slot.length|PT0.1S|Time to back off during login to the thrift frontend service.|duration|1.4.0
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 93e4f1317..5de90ea81 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -580,6 +580,17 @@ object KyuubiConf {
.booleanConf
.createWithDefault(true)
+ val FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER: ConfigEntry[String] =
+ buildConf("kyuubi.frontend.proxy.http.client.ip.header")
+ .doc("The http header to record the real client ip address. If your server is behind a load" +
+ " balancer or other proxy, the server will see this load balancer or proxy IP address as" +
+ " the client IP address, to get around this common issue, most load balancers or proxies" +
+ " offer the ability to record the real remote IP address in an HTTP herader that will be" +
+ " added to the request for other devices to use.")
+ .version("1.6.0")
+ .stringConf
+ .createWithDefault("X-Real-IP")
+
val AUTHENTICATION_METHOD: ConfigEntry[Seq[String]] = buildConf("kyuubi.authentication")
.doc("A comma separated list of client authentication types.<ul>" +
" <li>NOSASL: raw transport.</li>" +
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
index 005844071..f2d6116ec 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
@@ -33,6 +33,7 @@ import org.apache.thrift.server.TServlet
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
import org.apache.kyuubi.server.http.authentication.AuthenticationFilter
import org.apache.kyuubi.server.http.authentication.AuthenticationHandler.AUTHORIZATION_HEADER
import org.apache.kyuubi.server.http.util.{CookieSigner, HttpAuthUtils, SessionManager}
@@ -118,7 +119,8 @@ class ThriftHttpServlet(
val doAsQueryParam = getDoAsQueryParam(request.getQueryString)
if (doAsQueryParam != null) SessionManager.setProxyUserName(doAsQueryParam)
- clientIpAddress = request.getRemoteAddr
+ clientIpAddress = Option(request.getHeader(conf.get(FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER)))
+ .getOrElse(request.getRemoteAddr)
debug("Client IP Address: " + clientIpAddress)
// Set the thread local ip address
SessionManager.setIpAddress(clientIpAddress)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/authentication/AuthenticationFilter.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/authentication/AuthenticationFilter.scala
index b486a7f58..f76e068fd 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/authentication/AuthenticationFilter.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/authentication/AuthenticationFilter.scala
@@ -26,7 +26,7 @@ import scala.collection.mutable.HashMap
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.AUTHENTICATION_METHOD
+import org.apache.kyuubi.config.KyuubiConf.{AUTHENTICATION_METHOD, FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER}
import org.apache.kyuubi.service.authentication.{AuthTypes, InternalSecurityAccessor}
import org.apache.kyuubi.service.authentication.AuthTypes.{KERBEROS, NOSASL}
@@ -114,7 +114,8 @@ class AuthenticationFilter(conf: KyuubiConf) extends Filter with Logging {
HttpServletResponse.SC_UNAUTHORIZED,
s"No auth scheme matched for $authorization")
} else {
- HTTP_CLIENT_IP_ADDRESS.set(httpRequest.getRemoteAddr)
+ HTTP_CLIENT_IP_ADDRESS.set(Option(httpRequest.getHeader(
+ conf.get(FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER))).getOrElse(httpRequest.getRemoteAddr))
try {
val authUser = matchedHandler.authenticate(httpRequest, httpResponse)
if (authUser != null) {
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala
index 7c5a1aad1..7fdd7a7e2 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/BatchesResourceSuite.scala
@@ -566,4 +566,30 @@ class BatchesResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper {
assert(deleteResp.readEntity(classOf[CloseBatchResponse]).getMsg.contains(
s"Api request failed for http://${metadata2.kyuubiInstance}"))
}
+
+ test("support to get the real client ip for http proxy use case") {
+ val realClientIp = "localtest.me"
+
+ val sessionManager = fe.be.sessionManager.asInstanceOf[KyuubiSessionManager]
+ val appName = "spark-batch-submission"
+ val requestObj = new BatchRequest(
+ "spark",
+ sparkProcessBuilder.mainResource.get,
+ sparkProcessBuilder.mainClass,
+ appName,
+ Map(
+ "spark.master" -> "local",
+ s"spark.${ENGINE_SPARK_MAX_LIFETIME.key}" -> "5000",
+ s"spark.${ENGINE_CHECK_INTERVAL.key}" -> "1000").asJava,
+ Seq.empty[String].asJava)
+
+ val response = webTarget.path("api/v1/batches")
+ .request(MediaType.APPLICATION_JSON_TYPE)
+ .header(conf.get(FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER), realClientIp)
+ .post(Entity.entity(requestObj, MediaType.APPLICATION_JSON_TYPE))
+ assert(200 == response.getStatus)
+ val batch = response.readEntity(classOf[Batch])
+ val batchSession = sessionManager.getBatchSessionImpl(SessionHandle.fromUUID(batch.getId))
+ assert(batchSession.ipAddress === realClientIp)
+ }
}