You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2017/12/17 16:26:38 UTC
aurora git commit: Add metadata field to Job object in DSL
Repository: aurora
Updated Branches:
refs/heads/master 2e1ca4288 -> f1d9caf36
Add metadata field to Job object in DSL
Bugs closed: AURORA-1898
Reviewed at https://reviews.apache.org/r/64341/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/f1d9caf3
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/f1d9caf3
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/f1d9caf3
Branch: refs/heads/master
Commit: f1d9caf36dea2dbab6ccc44b9ba08a5572d7bbc8
Parents: 2e1ca42
Author: Jing Chen <mi...@gmail.com>
Authored: Sun Dec 17 08:26:33 2017 -0800
Committer: Bill Farner <wf...@apache.org>
Committed: Sun Dec 17 08:26:33 2017 -0800
----------------------------------------------------------------------
RELEASE-NOTES.md | 1 +
docs/reference/configuration.md | 9 +++++
.../python/apache/aurora/config/schema/base.py | 6 +++
src/main/python/apache/aurora/config/thrift.py | 10 ++++-
.../apache/aurora/client/cli/test_inspect.py | 1 +
.../python/apache/aurora/config/test_thrift.py | 41 ++++++++++++++++++++
6 files changed, 67 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 54dcc75..3053e54 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -47,6 +47,7 @@
- The scheduler no longer uses an internal H2 database for storage.
- There is a new Scheduler UI which, in addition to the facelift, provides the ability to inject your
own custom UI components.
+- Introduce a metadata field in the Job object of the DSL, which will populate TaskConfig.metadata.
### Deprecations and removals:
http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/docs/reference/configuration.md
----------------------------------------------------------------------
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index 67d9914..725e073 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -357,6 +357,7 @@ Job Schema
```announce``` | ```Announcer``` object | Optionally enable Zookeeper ServerSet announcements. See [Announcer Objects] for more information.
```enable_hooks``` | Boolean | Whether to enable [Client Hooks](client-hooks.md) for this job. (Default: False)
```partition_policy``` | ```PartitionPolicy``` object | An optional partition policy that allows job owners to define how to handle partitions for running tasks (in partition-aware Aurora clusters)
+ ```metadata``` | list of ```Metadata``` objects | list of ```Metadata``` objects for user's customized metadata information.
### UpdateConfig Objects
@@ -410,6 +411,14 @@ Parameters for controlling a task's health checks via HTTP or a shell command.
| ```reschedule``` | Boolean | Whether or not to reschedule when running tasks become partitioned (Default: True)
| ```delay_secs``` | Integer | How long to delay transitioning to LOST when running tasks are partitioned. (Default: 0)
+### Metadata Objects
+
+Describes a piece of user metadata in a key value pair
+
+ param | type | description
+ ----- | :----: | -----------
+ ```key``` | String | Indicate which metadata the user provides
+ ```value``` | String | Provide the metadata content for corresponding key
### Announcer Objects
http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/main/python/apache/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
index a466e78..3d57d6a 100644
--- a/src/main/python/apache/aurora/config/schema/base.py
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -159,6 +159,11 @@ class PartitionPolicy(Struct):
delay_secs = Default(Integer, 0)
+class Metadata(Struct):
+ key = Required(String)
+ value = Required(String)
+
+
class MesosJob(Struct):
name = Default(String, '{{task.name}}')
role = Required(String)
@@ -176,6 +181,7 @@ class MesosJob(Struct):
update_config = Default(UpdateConfig, UpdateConfig())
constraints = Map(String, String)
+ metadata = Default(List(Metadata), [])
service = Default(Boolean, False)
max_task_failures = Default(Integer, 1)
production = Default(Boolean, False)
http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/main/python/apache/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py
index eb00144..dcabb03 100644
--- a/src/main/python/apache/aurora/config/thrift.py
+++ b/src/main/python/apache/aurora/config/thrift.py
@@ -275,7 +275,15 @@ def convert(job, metadata=frozenset(), ports=frozenset()):
fully_interpolated(job.partition_policy().delay_secs()))
# Add metadata to a task, to display in the scheduler UI.
- task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata)
+ metadata_set = frozenset()
+ if job.has_metadata():
+ customized_metadata = job.metadata()
+ metadata_set |= frozenset(
+ (str(fully_interpolated(key_value_metadata.key())),
+ str(fully_interpolated(key_value_metadata.value())))
+ for key_value_metadata in customized_metadata)
+ metadata_set |= frozenset((str(key), str(value)) for key, value in metadata)
+ task.metadata = frozenset(Metadata(key=key, value=value) for key, value in metadata_set)
# task components
if not task_raw.has_resources():
http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/test/python/apache/aurora/client/cli/test_inspect.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_inspect.py b/src/test/python/apache/aurora/client/cli/test_inspect.py
index ecefc18..8c62480 100644
--- a/src/test/python/apache/aurora/client/cli/test_inspect.py
+++ b/src/test/python/apache/aurora/client/cli/test_inspect.py
@@ -138,6 +138,7 @@ Process 'process':
"production": False,
"role": "bozo",
"contact": "bozo@the.clown",
+ "metadata": [],
"lifecycle": {
"http": {
"graceful_shutdown_endpoint": "/quitquitquit",
http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/test/python/apache/aurora/config/test_thrift.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py
index 76d0ad6..7bf0508 100644
--- a/src/test/python/apache/aurora/config/test_thrift.py
+++ b/src/test/python/apache/aurora/config/test_thrift.py
@@ -27,6 +27,7 @@ from apache.aurora.config.schema.base import (
HealthCheckConfig,
Job,
Mesos,
+ Metadata,
Mode,
Parameter,
SimpleTask,
@@ -269,6 +270,46 @@ def test_metadata_in_config():
assert pi.value == '1'
+def test_config_with_metadata():
+ expected_metadata_tuples = frozenset([("city", "LA"), ("city", "SF")])
+ job = convert_pystachio_to_thrift(
+ HELLO_WORLD(metadata=[
+ Metadata(key=key, value=value)
+ for key, value in expected_metadata_tuples]))
+ tti = job.taskConfig
+
+ metadata_tuples = frozenset((key_value.key, key_value.value)
+ for key_value in tti.metadata)
+ assert metadata_tuples == expected_metadata_tuples
+
+
+def test_config_with_key_collision_metadata():
+ input_metadata_tuples = frozenset([("city", "LA")])
+ job = convert_pystachio_to_thrift(
+ HELLO_WORLD(metadata=[
+ Metadata(key=key, value=value)
+ for key, value in input_metadata_tuples]), metadata=[('city', "SF")])
+ tti = job.taskConfig
+
+ metadata_tuples = frozenset((key_value.key, key_value.value)
+ for key_value in tti.metadata)
+ expected_metadata_tuples = frozenset([("city", "LA"), ("city", "SF")])
+ assert metadata_tuples == expected_metadata_tuples
+
+
+def test_config_with_duplicate_metadata():
+ expected_metadata_tuples = frozenset([("city", "LA")])
+ job = convert_pystachio_to_thrift(
+ HELLO_WORLD(metadata=[
+ Metadata(key=key, value=value)
+ for key, value in expected_metadata_tuples]), metadata=[('city', "LA")])
+ tti = job.taskConfig
+
+ metadata_tuples = frozenset((key_value.key, key_value.value)
+ for key_value in tti.metadata)
+ assert metadata_tuples == expected_metadata_tuples
+
+
def test_task_instance_from_job():
instance = task_instance_from_job(
Job(health_check_config=HealthCheckConfig(interval_secs=30)), 0, '')