You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by se...@apache.org on 2016/10/06 20:15:10 UTC
aurora git commit: Use the Thrift binary protocol in the Aurora
client.
Repository: aurora
Updated Branches:
refs/heads/master 8cd175ad2 -> 09b8e583f
Use the Thrift binary protocol in the Aurora client.
Now that Aurora 0.16 has been released, we can assume all schedulers
are supporting the binary protocol.
Switching the protocol comes with a slight performance boost: Given
the example job `devcluster/www-data/prod/hello` with 500 instances
we can reduce the average request time by about 40%.
Using the TJSONProtocol:
$ time aurora job status devcluster > /dev/null
real 0m0.906s
user 0m0.759s
sys 0m0.078s
Using the new TBinaryProtocol:
$ time aurora job status devcluster > /dev/null
real 0m0.524s
user 0m0.410s
sys 0m0.079s
Reviewed at https://reviews.apache.org/r/52610/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/09b8e583
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/09b8e583
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/09b8e583
Branch: refs/heads/master
Commit: 09b8e583fc5f6314de3099af9896380dc18f691d
Parents: 8cd175a
Author: Stephan Erb <se...@apache.org>
Authored: Thu Oct 6 22:14:28 2016 +0200
Committer: Stephan Erb <se...@apache.org>
Committed: Thu Oct 6 22:14:28 2016 +0200
----------------------------------------------------------------------
RELEASE-NOTES.md | 1 +
.../aurora/client/api/scheduler_client.py | 4 ++--
.../python/apache/aurora/common/transport.py | 3 ++-
.../apache/aurora/common/test_transport.py | 20 ++++++++++----------
4 files changed, 15 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 6968bb5..f3dd8bb 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -8,6 +8,7 @@
`minWaitInInstanceRunningMs` will no longer have to be chosen based on the worst observed instance
startup/warmup delay but rather as a desired health check duration.
- A task's tier is now mapped to a label on the Mesos `TaskInfo` proto.
+- The Aurora client is now using the Thrift binary protocol to communicate with the scheduler.
### Deprecations and removals:
http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/src/main/python/apache/aurora/client/api/scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/scheduler_client.py b/src/main/python/apache/aurora/client/api/scheduler_client.py
index cbdb50a..9bbfece 100644
--- a/src/main/python/apache/aurora/client/api/scheduler_client.py
+++ b/src/main/python/apache/aurora/client/api/scheduler_client.py
@@ -19,7 +19,7 @@ import traceback
import requests
from pystachio import Default, Integer, String
-from thrift.protocol import TJSONProtocol
+from thrift.protocol import TBinaryProtocol
from thrift.transport import TTransport
from twitter.common import log
from twitter.common.concurrent import Timeout, deadline
@@ -111,7 +111,7 @@ class SchedulerClient(object):
_bypass_leader_redirect_session_factory,
should_bypass=self._bypass_leader_redirect))
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
schedulerClient = AuroraAdmin.Client(protocol)
for _ in range(self.THRIFT_RETRIES):
try:
http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/src/main/python/apache/aurora/common/transport.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/transport.py b/src/main/python/apache/aurora/common/transport.py
index 2e38847..f44d13a 100644
--- a/src/main/python/apache/aurora/common/transport.py
+++ b/src/main/python/apache/aurora/common/transport.py
@@ -117,7 +117,8 @@ class TRequestsTransport(TTransportBase):
data = self.__wbuf.getvalue()
self.__wbuf = BytesIO()
- self._session.headers['Content-Type'] = 'application/x-thrift'
+ self._session.headers['Accept'] = 'application/vnd.apache.thrift.binary'
+ self._session.headers['Content-Type'] = 'application/vnd.apache.thrift.binary'
self._session.headers['Content-Length'] = str(len(data))
self._session.headers['Host'] = self.__urlparse.hostname
http://git-wip-us.apache.org/repos/asf/aurora/blob/09b8e583/src/test/python/apache/aurora/common/test_transport.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/common/test_transport.py b/src/test/python/apache/aurora/common/test_transport.py
index 0a587f0..c3d35d2 100644
--- a/src/test/python/apache/aurora/common/test_transport.py
+++ b/src/test/python/apache/aurora/common/test_transport.py
@@ -19,7 +19,7 @@ import pytest
import requests
from mock import ANY, Mock, call, create_autospec
from requests import exceptions as request_exceptions
-from thrift.protocol import TJSONProtocol
+from thrift.protocol import TBinaryProtocol
from thrift.server import THttpServer
from thrift.transport import TTransport
@@ -38,7 +38,7 @@ class ReadOnlySchedulerHandler(object):
def test_request_transport_integration():
handler = ReadOnlySchedulerHandler()
processor = ReadOnlyScheduler.Processor(handler)
- pfactory = TJSONProtocol.TJSONProtocolFactory()
+ pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
server = THttpServer.THttpServer(processor, ('localhost', 0), pfactory)
server_thread = Thread(target=server.serve)
server_thread.start()
@@ -46,7 +46,7 @@ def test_request_transport_integration():
try:
transport = TRequestsTransport('http://localhost:%d' % server_port)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
response = client.getRoleSummary()
finally:
@@ -64,7 +64,7 @@ def test_request_transport_timeout():
session.headers = {}
session.post = Mock(side_effect=request_exceptions.Timeout())
transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
with pytest.raises(TTransport.TTransportException) as execinfo:
@@ -84,7 +84,7 @@ def test_raise_for_status_causes_exception():
session.post.return_value = response
transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
with pytest.raises(TTransport.TTransportException) as excinfo:
@@ -105,7 +105,7 @@ def test_raises_auth_error():
session.post.return_value = response
transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
with pytest.raises(TRequestsTransport.AuthError):
@@ -119,7 +119,7 @@ def test_request_any_other_exception():
session.headers = {}
session.post = Mock(side_effect=request_exceptions.ConnectionError())
transport = TRequestsTransport('http://localhost:12345', session_factory=lambda: session)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
with pytest.raises(TTransport.TTransportException):
@@ -161,7 +161,7 @@ def test_auth_type_valid():
auth = requests.auth.AuthBase()
transport = TRequestsTransport('http://localhost:1', auth=auth, session_factory=lambda: session)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
with pytest.raises(TTransport.TTransportException):
@@ -181,7 +181,7 @@ def test_auth_type_invalid():
def test_requests_transport_session_reuse():
handler = ReadOnlySchedulerHandler()
processor = ReadOnlyScheduler.Processor(handler)
- pfactory = TJSONProtocol.TJSONProtocolFactory()
+ pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
server = THttpServer.THttpServer(processor, ('localhost', 0), pfactory)
server_thread = Thread(target=server.serve)
server_thread.start()
@@ -189,7 +189,7 @@ def test_requests_transport_session_reuse():
try:
transport = TRequestsTransport('http://localhost:%d' % server_port)
- protocol = TJSONProtocol.TJSONProtocol(transport)
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
client.getRoleSummary()
old_session = transport._session