You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by se...@apache.org on 2016/03/18 20:15:17 UTC
aurora git commit: Allow overriding hostname before announcing
(https://issues.apache.org/jira/browse/AURORA-1611)
Repository: aurora
Updated Branches:
refs/heads/master 12be6fb72 -> efa292944
Allow overriding hostname before announcing (https://issues.apache.org/jira/browse/AURORA-1611)
Reviewed at https://reviews.apache.org/r/44532/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/efa29294
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/efa29294
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/efa29294
Branch: refs/heads/master
Commit: efa29294427d88af43c5f806f65136b097bd7625
Parents: 12be6fb
Author: Kunal Thakar <ku...@gmail.com>
Authored: Fri Mar 18 20:14:11 2016 +0100
Committer: Stephan Erb <st...@dev.static-void.de>
Committed: Fri Mar 18 20:14:11 2016 +0100
----------------------------------------------------------------------
RELEASE-NOTES.md | 2 ++
docs/configuration-reference.md | 24 ++++++++++++++++++++
.../executor/bin/thermos_executor_main.py | 11 ++++++++-
.../apache/aurora/executor/common/announcer.py | 24 +++++++++++++-------
.../aurora/executor/common/test_announcer.py | 20 +++++++++++++++-
5 files changed, 71 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/efa29294/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 42bb340..6e9364e 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -8,6 +8,8 @@
- Upgraded the scheduler ZooKeeper client from 3.4.6 to 3.4.8.
- Added support for dedicated constraints not exclusive to a particular role.
See [here](docs/deploying-aurora-scheduler.md#dedicated-attribute) for more details.
+- Added a new argument `--announcer-hostname` to thermos executor to override hostname in service
+ registry endpoint. See [here](docs/configuration-reference.md#announcer-objects) for details.
### Deprecations and removals:
http://git-wip-us.apache.org/repos/asf/aurora/blob/efa29294/docs/configuration-reference.md
----------------------------------------------------------------------
diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md
index e043c48..2362659 100644
--- a/docs/configuration-reference.md
+++ b/docs/configuration-reference.md
@@ -34,6 +34,7 @@ Aurora + Thermos Configuration Reference
- [Container Objects](#container)
- [LifecycleConfig Objects](#lifecycleconfig-objects)
- [Specifying Scheduling Constraints](#specifying-scheduling-constraints)
+- [Executor Wrapper](#executor-wrapper)
- [Template Namespaces](#template-namespaces)
- [mesos Namespace](#mesos-namespace)
- [thermos Namespace](#thermos-namespace)
@@ -461,6 +462,15 @@ zookeeper ensemble configured by the executor (which can be optionally overriden
zk_path parameter). If no Announcer object is specified,
no announcement will take place. For more information about ServerSets, see the [User Guide](user-guide.md).
+By default, the hostname in the registered endpoints will be the `--hostname` parameter
+that is passed to the mesos slave. To override the hostname value, the executor can be started
+with `--announcer-hostname=<overriden_value>`. If you decide to use `--announcer-hostname` and if
+the overriden value needs to change for every executor, then the executor has to be started inside a wrapper, see [Executor Wrapper](#executor-wrapper).
+
+For example, if you want the hostname in the endpoint to be an IP address instead of the hostname,
+the `--hostname` parameter to the mesos slave can be set to the machine IP or the executor can
+be started with `--announcer-hostname=<host_ip>` while wrapping the executor inside a script.
+
| object | type | description
| ------- | :-------: | --------
| ```primary_port``` | String | Which named port to register as the primary endpoint in the ServerSet (Default: `http`)
@@ -591,6 +601,20 @@ most one task per rack:
Use these constraints sparingly as they can dramatically reduce Tasks' schedulability.
+
+Executor Wrapper
+================
+
+If you need to do computation before starting the thermos executor (for example, setting a different
+`--announcer-hostname` parameter for every executor), then the thermos executor should be invoked
+ inside a wrapper script. In such a case, the aurora scheduler should be started with
+ `-thermos_executor_path` pointing to the wrapper script and `-thermos_executor_resources`
+ set to a comma separated string of all the resources that should be copied into
+ the sandbox (including the original thermos executor).
+
+For example, to wrap the executor inside a simple wrapper, the scheduler will be started like this
+`-thermos_executor_path=/path/to/wrapper.sh -thermos_executor_resources=/usr/share/aurora/bin/thermos_executor.pex`
+
Template Namespaces
===================
http://git-wip-us.apache.org/repos/asf/aurora/blob/efa29294/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
index f82858c..6634506 100644
--- a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
+++ b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
@@ -91,6 +91,14 @@ app.add_option(
default=False,
help='Allows setting arbitrary serverset path through the Announcer configuration.')
+app.add_option(
+ '--announcer-hostname',
+ dest='announcer_hostname',
+ type=str,
+ default=None,
+ help='Set hostname to be announced. By default it is'
+ 'the --hostname argument passed into the mesos slave'
+)
app.add_option(
'--execute-as-user',
@@ -186,7 +194,8 @@ def initialize(options):
status_providers.append(DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path,
- options.announcer_allow_custom_serverset_path
+ options.announcer_allow_custom_serverset_path,
+ options.announcer_hostname
))
# Create executor stub
http://git-wip-us.apache.org/repos/asf/aurora/blob/efa29294/src/main/python/apache/aurora/executor/common/announcer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/announcer.py b/src/main/python/apache/aurora/executor/common/announcer.py
index 34e36e0..79a9cfb 100644
--- a/src/main/python/apache/aurora/executor/common/announcer.py
+++ b/src/main/python/apache/aurora/executor/common/announcer.py
@@ -54,9 +54,10 @@ def make_endpoints(hostname, portmap, primary_port):
class AnnouncerCheckerProvider(StatusCheckerProvider):
- def __init__(self, allow_custom_serverset_path=False, name=None):
+ def __init__(self, allow_custom_serverset_path=False, hostname=None, name=None):
self.name = name
- self.__allow_custom_serverset_path = allow_custom_serverset_path
+ self._allow_custom_serverset_path = allow_custom_serverset_path
+ self._override_hostname = hostname
super(AnnouncerCheckerProvider, self).__init__()
@abstractmethod
@@ -75,17 +76,23 @@ class AnnouncerCheckerProvider(StatusCheckerProvider):
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
+ # Overriding hostname can be done either by explicitly specifying a value or
+ # by changing the value of assigned_task.slaveHost.
# assigned_task.slaveHost is the --hostname argument passed into the mesos slave.
- # Using this allows overriding the hostname published into ZK when announcing.
- # If no argument was passed to the mesos-slave, the slave falls back to gethostname().
+ # If no argument was passed to the mesos-slave, the slave falls back to gethostname()
+ if self._override_hostname:
+ hostname = self._override_hostname
+ else:
+ hostname = assigned_task.slaveHost
+
endpoint, additional = make_endpoints(
- assigned_task.slaveHost,
+ hostname,
portmap,
mesos_task.announce().primary_port().get())
client = self.make_zk_client()
if mesos_task.announce().has_zk_path():
- if self.__allow_custom_serverset_path:
+ if self._allow_custom_serverset_path:
path = mesos_task.announce().zk_path().get()
else:
app.error('Executor must be started with --announcer-allow-custom-serverset-path in order '
@@ -111,10 +118,11 @@ class DefaultAnnouncerCheckerProvider(AnnouncerCheckerProvider):
max_delay=DEFAULT_RETRY_MAX_DELAY.as_(Time.SECONDS),
)
- def __init__(self, ensemble, root='/aurora', allow_custom_serverset_path=False):
+ def __init__(self, ensemble, root='/aurora', allow_custom_serverset_path=False, hostname=None):
self.__ensemble = ensemble
self.__root = root
- super(DefaultAnnouncerCheckerProvider, self).__init__(allow_custom_serverset_path)
+ super(DefaultAnnouncerCheckerProvider, self).__init__(
+ allow_custom_serverset_path, hostname)
def make_zk_client(self):
return KazooClient(self.__ensemble, connection_retry=self.DEFAULT_RETRY_POLICY)
http://git-wip-us.apache.org/repos/asf/aurora/blob/efa29294/src/test/python/apache/aurora/executor/common/test_announcer.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/executor/common/test_announcer.py b/src/test/python/apache/aurora/executor/common/test_announcer.py
index f4032c7..142b58d 100644
--- a/src/test/python/apache/aurora/executor/common/test_announcer.py
+++ b/src/test/python/apache/aurora/executor/common/test_announcer.py
@@ -17,7 +17,7 @@ import threading
import pytest
from kazoo.client import KazooClient
from kazoo.exceptions import KazooException
-from mock import MagicMock, create_autospec, patch
+from mock import MagicMock, call, create_autospec, patch
from twitter.common.quantity import Amount, Time
from twitter.common.testing.clock import ThreadedClock
from twitter.common.zookeeper.serverset import Endpoint, ServerSet
@@ -321,3 +321,21 @@ def test_announcer_provider_with_zkpath(mock_client_provider, mock_serverset_pro
mock_serverset_provider.assert_called_once_with(mock_client, '/uns/v1/sjc1-prod/us1/service/prod')
assert checker.name() == 'announcer'
assert checker.status is None
+
+
+@patch('apache.aurora.executor.common.announcer.ServerSet')
+@patch('apache.aurora.executor.common.announcer.KazooClient')
+@patch('apache.aurora.executor.common.announcer.Endpoint')
+def test_announcer_provider_with_hostcmd(endpoint_mock_provider,
+ mock_client_provider, mock_serverset_provider):
+ mock_client = create_autospec(spec=KazooClient, instance=True)
+ mock_client_provider.return_value = mock_client
+ mock_serverset = create_autospec(spec=ServerSet, instance=True)
+ mock_serverset_provider.return_value = mock_serverset
+
+ dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', '/aurora', False, '10.2.3.4')
+ job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={})
+ assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
+ dap.from_assigned_task(assigned_task, None)
+
+ assert endpoint_mock_provider.mock_calls == [call('10.2.3.4', 12345), call('10.2.3.4', 12345)]