You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ji...@apache.org on 2026/10/10 09:25:08 UTC
(ambari-mpacks) 24/35: AMBARI-26663: Render native content and stage validated configuration files
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ambari-mpacks.git
commit 3c78450b0b373b4c666669128bee4bbc4794fcfa
Author: jialiang <ji...@gmail.com>
AuthorDate: Mon Sep 28 17:44:09 2026 +0800
AMBARI-26663: Render native content and stage validated configuration files
---
mpacks/airflow/mpack.json | 2 +-
mpacks/airflow/shared-files.json | 7 +-
mpacks/celeborn/mpack.json | 2 +-
mpacks/celeborn/shared-files.json | 7 +-
mpacks/dolphinscheduler/mpack.json | 2 +-
mpacks/dolphinscheduler/shared-files.json | 7 +-
mpacks/doris/mpack.json | 2 +-
mpacks/doris/shared-files.json | 7 +-
mpacks/elasticsearch/mpack.json | 2 +-
mpacks/elasticsearch/shared-files.json | 7 +-
mpacks/kyuubi/mpack.json | 2 +-
mpacks/kyuubi/shared-files.json | 7 +-
mpacks/minio/mpack.json | 2 +-
mpacks/minio/shared-files.json | 7 +-
mpacks/nginx/mpack.json | 2 +-
mpacks/nginx/shared-files.json | 15 +++
mpacks/postgresql/mpack.json | 2 +-
mpacks/postgresql/shared-files.json | 15 +++
mpacks/trino/mpack.json | 2 +-
mpacks/trino/shared-files.json | 7 +-
release.json | 20 ++--
shared/content_files.py | 123 +++++++++++++++++++++
shared/content_formats.py | 171 ++++++++++++++++++++++++++++++
shared/native_config.py | 35 +++++-
tests/test_content_files.py | 135 +++++++++++++++++++++++
tests/test_new_release.py | 4 +-
26 files changed, 561 insertions(+), 33 deletions(-)
diff --git a/mpacks/airflow/mpack.json b/mpacks/airflow/mpack.json
index 0d58492..a70a761 100644
--- a/mpacks/airflow/mpack.json
+++ b/mpacks/airflow/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "airflow",
- "version": "1.0.0.6",
+ "version": "1.0.1.0",
"description": "Apache Airflow basic topology",
"artifacts": [
{
diff --git a/mpacks/airflow/shared-files.json b/mpacks/airflow/shared-files.json
index 5adc5ac..273497f 100644
--- a/mpacks/airflow/shared-files.json
+++ b/mpacks/airflow/shared-files.json
@@ -20,7 +20,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/AIRFLOW/1.0/services/AIRFLOW/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/AIRFLOW/1.0/services/AIRFLOW/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/celeborn/mpack.json b/mpacks/celeborn/mpack.json
index 97a04a3..403753e 100644
--- a/mpacks/celeborn/mpack.json
+++ b/mpacks/celeborn/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "celeborn",
- "version": "1.0.0.2",
+ "version": "1.0.1.0",
"description": "Apache Celeborn basic topology",
"artifacts": [
{
diff --git a/mpacks/celeborn/shared-files.json b/mpacks/celeborn/shared-files.json
index a739557..cc73d9d 100644
--- a/mpacks/celeborn/shared-files.json
+++ b/mpacks/celeborn/shared-files.json
@@ -20,7 +20,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/CELEBORN/1.0/services/CELEBORN/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/CELEBORN/1.0/services/CELEBORN/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/dolphinscheduler/mpack.json b/mpacks/dolphinscheduler/mpack.json
index 9bd223f..e4320b6 100644
--- a/mpacks/dolphinscheduler/mpack.json
+++ b/mpacks/dolphinscheduler/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "dolphinscheduler",
- "version": "1.0.0.5",
+ "version": "1.0.1.0",
"description": "DolphinScheduler 3.1.9 persistent pseudo-cluster on BIGTOP",
"artifacts": [{"name": "dolphinscheduler-definitions", "type": "extension-definitions", "source_dir": "extensions"}],
"dependencies": []
diff --git a/mpacks/dolphinscheduler/shared-files.json b/mpacks/dolphinscheduler/shared-files.json
index f4866bf..7a066f2 100644
--- a/mpacks/dolphinscheduler/shared-files.json
+++ b/mpacks/dolphinscheduler/shared-files.json
@@ -35,7 +35,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/DOLPHINSCHEDULER/1.0/services/DOLPHINSCHEDULER/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/DOLPHINSCHEDULER/1.0/services/DOLPHINSCHEDULER/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/doris/mpack.json b/mpacks/doris/mpack.json
index a5eee65..99e09e6 100644
--- a/mpacks/doris/mpack.json
+++ b/mpacks/doris/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "doris",
- "version": "1.0.0.7",
+ "version": "1.0.1.0",
"description": "Apache Doris 4.1.4 single FE and BE",
"artifacts": [{"name": "doris-definitions", "type": "extension-definitions", "source_dir": "extensions"}],
"dependencies": []
diff --git a/mpacks/doris/shared-files.json b/mpacks/doris/shared-files.json
index 804529b..f3bae5e 100644
--- a/mpacks/doris/shared-files.json
+++ b/mpacks/doris/shared-files.json
@@ -35,7 +35,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/DORIS/1.0/services/DORIS/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/DORIS/1.0/services/DORIS/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/elasticsearch/mpack.json b/mpacks/elasticsearch/mpack.json
index 7cd21d7..42e8407 100644
--- a/mpacks/elasticsearch/mpack.json
+++ b/mpacks/elasticsearch/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "elasticsearch",
- "version": "1.0.0.5",
+ "version": "1.0.1.0",
"description": "Elasticsearch 9.5.4 secure single-node deployment",
"artifacts": [{"name": "elasticsearch-definitions", "type": "extension-definitions", "source_dir": "extensions"}],
"dependencies": [{"name": "generic-base", "version": "1.0.0.3"}]
diff --git a/mpacks/elasticsearch/shared-files.json b/mpacks/elasticsearch/shared-files.json
index c489280..e2d60a0 100644
--- a/mpacks/elasticsearch/shared-files.json
+++ b/mpacks/elasticsearch/shared-files.json
@@ -35,7 +35,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/ELASTICSEARCH/1.0/services/ELASTICSEARCH/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/ELASTICSEARCH/1.0/services/ELASTICSEARCH/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/kyuubi/mpack.json b/mpacks/kyuubi/mpack.json
index 16929b9..fb7b150 100644
--- a/mpacks/kyuubi/mpack.json
+++ b/mpacks/kyuubi/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "kyuubi",
- "version": "1.0.0.11",
+ "version": "1.0.1.0",
"artifacts": [
{
"name": "kyuubi-definitions",
diff --git a/mpacks/kyuubi/shared-files.json b/mpacks/kyuubi/shared-files.json
index ac728b0..754974f 100644
--- a/mpacks/kyuubi/shared-files.json
+++ b/mpacks/kyuubi/shared-files.json
@@ -10,7 +10,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/KYUUBI/1.0/services/KYUUBI/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/KYUUBI/1.0/services/KYUUBI/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/minio/mpack.json b/mpacks/minio/mpack.json
index fff31c6..a32081e 100644
--- a/mpacks/minio/mpack.json
+++ b/mpacks/minio/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "minio",
- "version": "1.0.0.5",
+ "version": "1.0.1.0",
"description": "MinIO source-built single-node object storage",
"artifacts": [{"name": "minio-definitions", "type": "extension-definitions", "source_dir": "extensions"}],
"dependencies": [{"name": "generic-base", "version": "1.0.0.3"}]
diff --git a/mpacks/minio/shared-files.json b/mpacks/minio/shared-files.json
index 804d5ed..fc6aabc 100644
--- a/mpacks/minio/shared-files.json
+++ b/mpacks/minio/shared-files.json
@@ -35,7 +35,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/MINIO/1.0/services/MINIO/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/MINIO/1.0/services/MINIO/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/mpacks/nginx/mpack.json b/mpacks/nginx/mpack.json
index aaf3210..eaddbe9 100644
--- a/mpacks/nginx/mpack.json
+++ b/mpacks/nginx/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "nginx",
- "version": "1.0.0.4",
+ "version": "1.0.1.1",
"artifacts": [
{
"name": "nginx-definitions",
diff --git a/mpacks/nginx/shared-files.json b/mpacks/nginx/shared-files.json
index b946a5e..56f8977 100644
--- a/mpacks/nginx/shared-files.json
+++ b/mpacks/nginx/shared-files.json
@@ -6,6 +6,21 @@
"source": "shared/observations.py",
"target": "extensions/NGINX/1.0/services/NGINX/package/scripts/observations.py",
"digest": "8fb22532d4012faf25c2f38a56bbd713738016716fa6c0606fb56b46ce194a99"
+ },
+ {
+ "source": "shared/native_config.py",
+ "target": "extensions/NGINX/1.0/services/NGINX/package/scripts/native_config.py",
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/NGINX/1.0/services/NGINX/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
+ },
+ {
+ "source": "shared/content_files.py",
+ "target": "extensions/NGINX/1.0/services/NGINX/package/scripts/content_files.py",
+ "digest": "cadfb727f3d2d71ea37950b216a9a80792730e1026ec3a1d7510386f4d264b6b"
}
]
}
diff --git a/mpacks/postgresql/mpack.json b/mpacks/postgresql/mpack.json
index 6ccf553..b5d4f18 100644
--- a/mpacks/postgresql/mpack.json
+++ b/mpacks/postgresql/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "postgresql",
- "version": "1.0.0.5",
+ "version": "1.0.1.1",
"artifacts": [
{
"name": "postgresql-definitions",
diff --git a/mpacks/postgresql/shared-files.json b/mpacks/postgresql/shared-files.json
index de19f82..52f4fea 100644
--- a/mpacks/postgresql/shared-files.json
+++ b/mpacks/postgresql/shared-files.json
@@ -6,6 +6,21 @@
"source": "shared/observations.py",
"target": "extensions/POSTGRESQL/1.0/services/POSTGRESQL/package/scripts/observations.py",
"digest": "8fb22532d4012faf25c2f38a56bbd713738016716fa6c0606fb56b46ce194a99"
+ },
+ {
+ "source": "shared/native_config.py",
+ "target": "extensions/POSTGRESQL/1.0/services/POSTGRESQL/package/scripts/native_config.py",
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/POSTGRESQL/1.0/services/POSTGRESQL/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
+ },
+ {
+ "source": "shared/content_files.py",
+ "target": "extensions/POSTGRESQL/1.0/services/POSTGRESQL/package/scripts/content_files.py",
+ "digest": "cadfb727f3d2d71ea37950b216a9a80792730e1026ec3a1d7510386f4d264b6b"
}
]
}
diff --git a/mpacks/trino/mpack.json b/mpacks/trino/mpack.json
index b6039a4..4c7e65a 100644
--- a/mpacks/trino/mpack.json
+++ b/mpacks/trino/mpack.json
@@ -3,7 +3,7 @@
"schema_version": 1,
"type": "full-release",
"name": "trino",
- "version": "1.0.0.4",
+ "version": "1.0.1.0",
"description": "Trino 483 coordinator and optional workers on BIGTOP",
"artifacts": [{"name": "trino-definitions", "type": "extension-definitions", "source_dir": "extensions"}],
"dependencies": []
diff --git a/mpacks/trino/shared-files.json b/mpacks/trino/shared-files.json
index bac1b5e..c27c132 100644
--- a/mpacks/trino/shared-files.json
+++ b/mpacks/trino/shared-files.json
@@ -35,7 +35,12 @@
{
"source": "shared/native_config.py",
"target": "extensions/TRINO/1.0/services/TRINO/package/scripts/native_config.py",
- "digest": "897599c88c2fbbe3d238e02b519ae1b89c09f57bada53a688e216ac79484308e"
+ "digest": "6e32076f70695e572a73d76009e1d1cc862d93d5f44ca59700b9894b27203b02"
+ },
+ {
+ "source": "shared/content_formats.py",
+ "target": "extensions/TRINO/1.0/services/TRINO/package/scripts/content_formats.py",
+ "digest": "75400402680d22b12a5e72d936eaa2e1c7ed0dcc442b6ccaa572e6251483a331"
}
]
}
diff --git a/release.json b/release.json
index 1396c84..107077d 100644
--- a/release.json
+++ b/release.json
@@ -8,43 +8,43 @@
},
"nginx": {
"path": "mpacks/nginx",
- "version": "1.0.0.4"
+ "version": "1.0.1.1"
},
"postgresql": {
"path": "mpacks/postgresql",
- "version": "1.0.0.5"
+ "version": "1.0.1.1"
},
"kyuubi": {
"path": "mpacks/kyuubi",
- "version": "1.0.0.11"
+ "version": "1.0.1.0"
},
"airflow": {
"path": "mpacks/airflow",
- "version": "1.0.0.6"
+ "version": "1.0.1.0"
},
"celeborn": {
"path": "mpacks/celeborn",
- "version": "1.0.0.2"
+ "version": "1.0.1.0"
},
"dolphinscheduler": {
"path": "mpacks/dolphinscheduler",
- "version": "1.0.0.5"
+ "version": "1.0.1.0"
},
"trino": {
"path": "mpacks/trino",
- "version": "1.0.0.4"
+ "version": "1.0.1.0"
},
"doris": {
"path": "mpacks/doris",
- "version": "1.0.0.7"
+ "version": "1.0.1.0"
},
"elasticsearch": {
"path": "mpacks/elasticsearch",
- "version": "1.0.0.5"
+ "version": "1.0.1.0"
},
"minio": {
"path": "mpacks/minio",
- "version": "1.0.0.5"
+ "version": "1.0.1.0"
}
}
}
diff --git a/shared/content_files.py b/shared/content_files.py
new file mode 100644
index 0000000..4f92174
--- /dev/null
+++ b/shared/content_files.py
@@ -0,0 +1,123 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import fcntl
+import hashlib
+import os
+import pwd
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+from native_config import ConfigurationError, section
+
+
+def render(parameters, kind, variables):
+ values = section(parameters, kind)
+ if not values:
+ return None
+ if set(values) != {"content"} or not values["content"].strip():
+ raise ConfigurationError("A nonempty content property is required: " + kind)
+ value = values["content"]
+ for key, replacement in variables.items():
+ value = value.replace("{{" + key + "}}", str(replacement))
+ return value if value.endswith("\n") else value + "\n"
+
+
+def safe_path(path):
+ if any(item.is_symlink() for item in (path,) + tuple(path.parents)):
+ raise ConfigurationError("Configuration paths must not traverse symbolic links")
+
+
+def replace(path, data, uid, gid, mode):
+ safe_path(path)
+ fd, temporary = tempfile.mkstemp(prefix=".ambari-write-", dir=str(path.parent))
+ try:
+ with os.fdopen(fd, "wb") as stream:
+ os.fchmod(stream.fileno(), mode)
+ os.fchown(stream.fileno(), uid, gid)
+ stream.write(data)
+ stream.flush()
+ os.fsync(stream.fileno())
+ os.replace(temporary, str(path))
+ finally:
+ if os.path.exists(temporary):
+ os.unlink(temporary)
+
+
+@contextmanager
+def transaction(root, contents, validate, user):
+ """Validate staged files, replace atomically, and restore on activation failure."""
+ root = Path(root)
+ safe_path(root)
+ account = pwd.getpwnam(user)
+ lock = root / ".ambari-content.lock"
+ safe_path(lock)
+ with lock.open("a") as guard:
+ fcntl.flock(guard.fileno(), fcntl.LOCK_EX)
+ with tempfile.TemporaryDirectory(prefix=".ambari-config-", dir=str(root)) as temporary:
+ stage = Path(temporary)
+ os.chown(str(stage), account.pw_uid, account.pw_gid)
+ # Native parsers resolve relative includes beside the candidate file.
+ # Mirror untouched resources without copying or modifying their data.
+ for existing in root.iterdir():
+ if existing.name not in contents and not existing.name.startswith(".ambari"):
+ (stage / existing.name).symlink_to(existing)
+ previous, staged = {}, {}
+ for name, value in contents.items():
+ if Path(name).name != name or name.startswith("."):
+ raise ConfigurationError("Invalid managed configuration filename")
+ path = root / name
+ safe_path(path)
+ if path.exists():
+ status = path.stat()
+ previous[name] = (path.read_bytes(), status.st_uid, status.st_gid, status.st_mode & 0o777)
+ else:
+ previous[name] = None
+ staged[name] = stage / name
+ replace(staged[name], value.encode("utf-8"), account.pw_uid, account.pw_gid, 0o600)
+ validate(staged)
+ changed = []
+ try:
+ for name, value in contents.items():
+ current = root / name
+ if (previous[name] is None and current.exists() or previous[name] is not None and
+ (not current.exists() or current.read_bytes() != previous[name][0])):
+ raise ConfigurationError("Configuration changed concurrently; retry after reconciliation")
+ # Retain the immediately preceding file for manual crash recovery.
+ if previous[name] is not None:
+ replace(root / (name + ".ambari-before"), previous[name][0], account.pw_uid, account.pw_gid, 0o600)
+ changed.append(name)
+ replace(root / name, value.encode("utf-8"), account.pw_uid, account.pw_gid, 0o600)
+ yield {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in contents}
+ except BaseException:
+ for name in changed:
+ target = root / name
+ if target.exists() and target.read_bytes() not in (
+ contents[name].encode("utf-8"), previous[name][0] if previous[name] else b""):
+ raise ConfigurationError("Configuration changed during activation; retain the backup for reconciliation")
+ for name in reversed(changed):
+ if previous[name] is None:
+ if (root / name).exists():
+ (root / name).unlink()
+ else:
+ replace(root / name, *previous[name])
+ raise
+
+
+def result(request, outcome, observations):
+ return dict({key: request[key] for key in
+ ("schema_version", "definition_snapshot", "execution_id", "task_id", "action")},
+ outcome=outcome, observations=observations)
diff --git a/shared/content_formats.py b/shared/content_formats.py
new file mode 100644
index 0000000..9ce71b3
--- /dev/null
+++ b/shared/content_formats.py
@@ -0,0 +1,171 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import configparser
+import re
+import shlex
+
+
+class Document(dict):
+ def __init__(self, values, source, format_name):
+ super().__init__(values)
+ self.source = source
+ self.format_name = format_name
+ self.explicit = set(values)
+
+
+def unescape(value):
+ result, index = [], 0
+ while index < len(value):
+ char = value[index]
+ index += 1
+ if char == "\\" and index < len(value):
+ char = value[index]
+ index += 1
+ if char == "u":
+ digits = value[index:index + 4]
+ if not re.fullmatch(r"[0-9a-fA-F]{4}", digits):
+ raise ValueError("Invalid Unicode escape in properties content")
+ char = chr(int(digits, 16))
+ index += 4
+ else:
+ char = {"n": "\n", "r": "\r", "t": "\t", "f": "\f"}.get(char, char)
+ result.append(char)
+ return "".join(result).encode("utf-16", "surrogatepass").decode("utf-16")
+
+
+def parse_properties(source):
+ values, pending = {}, ""
+ for physical in source.splitlines():
+ line = pending + physical.lstrip(" \t\f")
+ if not pending and (not line or line.startswith(("#", "!"))):
+ continue
+ if (len(line) - len(line.rstrip("\\"))) % 2:
+ pending = line[:-1]
+ continue
+ pending = ""
+ escaped, boundary = False, len(line)
+ for index, char in enumerate(line):
+ if not escaped and char in "=: \t\f":
+ boundary = index
+ break
+ escaped = not escaped if char == "\\" else False
+ tail = line[boundary:].lstrip(" \t\f")
+ if tail.startswith(("=", ":")):
+ tail = tail[1:].lstrip(" \t\f")
+ values[unescape(line[:boundary])] = unescape(tail)
+ if pending:
+ raise ValueError("Incomplete continuation in properties content")
+ return Document(values, source, "properties")
+
+
+def parse_environment(source):
+ values = {}
+ for line in source.splitlines():
+ if not line.strip() or line.lstrip().startswith("#"):
+ continue
+ match = re.fullmatch(r"\s*(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=(.*)", line)
+ if not match:
+ raise ValueError("Environment content requires NAME=value assignments")
+ key, value = match.groups()
+ parts = shlex.split(value, comments=True)
+ if len(parts) > 1:
+ raise ValueError("Quote environment values containing spaces")
+ values[key] = parts[0] if parts else ""
+ return Document(values, source, "environment")
+
+
+def without_properties(source, keys):
+ result, pending = [], ""
+ for line in source.splitlines(keepends=True):
+ pending += line
+ physical = line.rstrip("\r\n")
+ if (len(physical) - len(physical.rstrip("\\"))) % 2:
+ continue
+ if not set(parse_properties(pending)) & set(keys):
+ result.append(pending)
+ pending = ""
+ if pending:
+ raise ValueError("Incomplete continuation in properties content")
+ return "".join(result)
+
+
+def parse_yaml(source):
+ import yaml
+ class UniqueLoader(yaml.SafeLoader):
+ pass
+ def mapping(loader, node, deep=False):
+ result = {}
+ for key_node, value_node in node.value:
+ key = loader.construct_object(key_node, deep=deep)
+ if key in result:
+ raise ValueError("Duplicate YAML configuration key")
+ result[key] = loader.construct_object(value_node, deep=deep)
+ return result
+ UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping)
+ data = yaml.load(source, Loader=UniqueLoader)
+ if not isinstance(data, dict):
+ raise ValueError("YAML configuration must be a mapping")
+ import json
+ flattened = {}
+ def flatten(node, prefix=""):
+ for key, value in node.items():
+ if not isinstance(key, str):
+ raise ValueError("YAML setting names must be strings")
+ name = prefix + key
+ if isinstance(value, dict):
+ flatten(value, name + ".")
+ else:
+ if name in flattened:
+ raise ValueError("Ambiguous nested and dotted YAML setting")
+ flattened[name] = value if isinstance(value, str) else json.dumps(value, allow_nan=False)
+ flatten(data)
+ return Document(flattened, source, "yaml")
+
+
+def parse_ini(source):
+ parser = configparser.ConfigParser(interpolation=None)
+ parser.read_string(source)
+ if parser.defaults():
+ raise ValueError("Use explicit INI sections instead of DEFAULT")
+ return Document({group + "." + key: value for group in parser.sections()
+ for key, value in parser.items(group)}, source, "ini")
+
+
+def preserve(document, rendered_missing):
+ source = document.source if document.source.endswith("\n") else document.source + "\n"
+ return source + (
+ "\n# Ambari managed settings and defaults\n" + rendered_missing if rendered_missing else "")
+
+
+def render_ini(document, values):
+ missing = {key: value for key, value in values.items() if key not in document.explicit}
+ result, current = [], None
+ def finish(group):
+ for key in list(missing):
+ section, option = key.split(".", 1)
+ if section == group:
+ result.append(option + " = " + str(missing.pop(key)))
+ for line in document.source.splitlines():
+ match = configparser.ConfigParser.SECTCRE.match(line.strip())
+ if match:
+ finish(current)
+ current = match.group("header")
+ result.append(line)
+ finish(current)
+ for group in sorted({key.split(".", 1)[0] for key in missing}):
+ result.extend(["", "[" + group + "]"])
+ finish(group)
+ return "\n".join(result) + "\n"
diff --git a/shared/native_config.py b/shared/native_config.py
index 78d6c40..485fe87 100644
--- a/shared/native_config.py
+++ b/shared/native_config.py
@@ -63,18 +63,36 @@ def validate_properties(values, name, multiline=False):
def properties(parameters, name, managed=None, defaults=None, reserved=()):
values = section(parameters, name)
- validate_properties(values, name)
+ from content_formats import Document, parse_environment, parse_properties, parse_yaml
+ if "content" in values:
+ # Content is authoritative; old scalar properties are migration inputs only.
+ source = values["content"]
+ parser = parse_yaml if name == "elasticsearch-settings" else parse_environment if name in {
+ "kyuubi-environment", "celeborn-environment", "minio-environment"} else parse_properties
+ try:
+ values = parser(source)
+ except Exception:
+ raise ConfigurationError("Invalid native file content: " + name)
+ validate_properties(values, name, multiline=isinstance(values, Document))
managed = {key: str(value) for key, value in (managed or {}).items()}
for key, value in values.items():
- if key.startswith(tuple(reserved)) or key in managed and value != managed[key]:
+ if key.startswith(tuple(reserved)) and key not in managed or key in managed and value != managed[key]:
raise ConfigurationError("Property is managed by the basic settings: " + name + "/" + key)
result = {key: str(value) for key, value in (defaults or {}).items()}
result.update(values)
result.update(managed)
+ if isinstance(values, Document):
+ document = Document(result, values.source, values.format_name)
+ document.explicit = values.explicit
+ return document
return result
def render_properties(values):
+ from content_formats import Document, preserve
+ if isinstance(values, Document):
+ return preserve(values, render_properties({key: value for key, value in values.items()
+ if key not in values.explicit}))
validate_properties(values, "properties")
def escape(value):
value = value.replace("\\", "\\\\").replace("\t", "\\t").replace(" ", "\\ ")
@@ -87,6 +105,9 @@ def render_properties(values):
def overlay_properties(original, values):
+ from content_formats import Document
+ if isinstance(values, Document):
+ return render_properties(values)
lines, skipping = [], False
for line in original.splitlines():
if not skipping:
@@ -135,10 +156,18 @@ def environment(parameters, name, managed=None):
def shell_exports(values):
- return "".join("export " + key + "=" + shlex.quote(value) + "\n" for key, value in sorted(values.items()))
+ from content_formats import Document
+ comments = ""
+ if isinstance(values, Document):
+ comments = "\n".join(line for line in values.source.splitlines() if not line.strip() or line.lstrip().startswith("#")) + "\n"
+ return comments + "".join("export " + key + "=" + shlex.quote(value) + "\n" for key, value in sorted(values.items()))
def yaml_properties(values):
+ from content_formats import Document, preserve
+ if isinstance(values, Document):
+ return preserve(values, yaml_properties({key: value for key, value in values.items()
+ if key not in values.explicit}))
# JSON scalar/array notation is valid YAML and avoids implicit YAML coercions.
result = []
for key, value in sorted(values.items()):
diff --git a/tests/test_content_files.py b/tests/test_content_files.py
new file mode 100644
index 0000000..8af3cf9
--- /dev/null
+++ b/tests/test_content_files.py
@@ -0,0 +1,135 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+import sys
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "shared"))
+from content_formats import parse_properties, parse_environment, parse_ini, render_ini
+from content_files import transaction, render, result
+from native_config import ConfigurationError, properties, render_properties, yaml_properties
+
+
+def params(kind, text, **extra):
+ return {"native_configs": {"schema_version": 1, "types": {kind: dict(extra, content=text)}}}
+
+
+class ContentFileTests(unittest.TestCase):
+ def test_content_preserves_trailing_escaped_spaces_and_blank_lines(self):
+ source = "# comment\nvalue=trailing\\ \n\n"
+ values = properties(params("site", source), "site")
+ self.assertEqual(values["value"], "trailing ")
+ self.assertEqual(render_properties(values), source)
+ self.assertEqual(render(params("nginx-conf", "# comment \n\n"), "nginx-conf", {}), "# comment \n\n")
+
+ def test_properties_support_comments_continuations_escapes_and_last_value(self):
+ source = "# keep comment\nx=first\nx : last\\ value\nurl=jdbc:postgresql://host/db\nlong=one\\\n two\nunicode=\\u4e2d\n"
+ value = parse_properties(source)
+ self.assertEqual(value["x"], "last value")
+ self.assertEqual(value["long"], "onetwo")
+ self.assertEqual(value["unicode"], "\u4e2d")
+ self.assertEqual(value["url"], "jdbc:postgresql://host/db")
+ rendered = render_properties(properties(params("trino-config", source, x="obsolete"), "trino-config", {"coordinator": "true"}))
+ self.assertTrue(rendered.startswith(source))
+ self.assertIn("coordinator=true\n", rendered)
+ self.assertNotIn("obsolete", rendered)
+
+ def test_managed_conflicts_are_explicit_and_matching_values_are_allowed(self):
+ source = "# exact user document\nhttp.port=9200\n"
+ value = properties(params("test", source), "test", {"http.port": 9200}, reserved=("http.",))
+ self.assertEqual(render_properties(value), source)
+ with self.assertRaises(ConfigurationError):
+ properties(params("test", "http.port=9201"), "test", {"http.port": 9200})
+
+ def test_ini_preserves_comments_and_adds_missing_managed_values(self):
+ source = "# heading\n[core]\n# user comment\nload_examples = False\n\n[logging]\nlogging_level = INFO\n"
+ doc = parse_ini(source)
+ values = dict(doc, **{"core.executor": "LocalExecutor", "database.sql_alchemy_conn": "postgresql://local"})
+ output = render_ini(doc, values)
+ self.assertIn("# user comment\nload_examples = False", output)
+ self.assertEqual(parse_ini(output), values)
+
+ def test_environment_values_are_literal_and_command_lines_are_rejected(self):
+ values = parse_environment("# note\nJAVA_OPTS='-Xmx1g -Dname=$NAME'\nEMPTY=\n")
+ self.assertEqual(values["JAVA_OPTS"], "-Xmx1g -Dname=$NAME")
+ self.assertEqual(values["EMPTY"], "")
+ with self.assertRaises(ValueError):
+ parse_environment("run-some-command\n")
+
+ def test_invalid_yaml_and_duplicate_keys_are_rejected(self):
+ try:
+ import yaml
+ except ImportError:
+ self.skipTest("PyYAML is required for YAML content verification")
+ source = "# keep YAML\nsearch:\n max_buckets: 1000\nnode.attr.rack: 'rack: 1'\n"
+ doc = properties(params("elasticsearch-settings", source), "elasticsearch-settings", {"http.port": 9200})
+ self.assertTrue(yaml_properties(doc).startswith(source))
+ self.assertEqual(yaml.safe_load(yaml_properties(doc))["http.port"], 9200)
+ for bad in ("x: 1\nx: 2\n", "not a mapping", "x: ["):
+ with self.subTest(bad=bad), self.assertRaises(ConfigurationError):
+ properties(params("elasticsearch-settings", bad), "elasticsearch-settings")
+
+ def test_native_validation_precedes_replacement_and_activation_failure_restores_files(self):
+ account = SimpleNamespace(pw_uid=os.getuid(), pw_gid=os.getgid())
+ with tempfile.TemporaryDirectory() as directory, patch("pwd.getpwnam", return_value=account):
+ root = Path(directory).resolve()
+ (root / "config.conf").write_text("original\n")
+ (root / "config.conf").chmod(0o640)
+ (root / "extra.conf").write_text("included=value\n")
+ def reject(staged):
+ self.assertEqual(staged["config.conf"].read_text(), "invalid\n")
+ self.assertEqual((staged["config.conf"].parent / "extra.conf").read_text(), "included=value\n")
+ self.assertEqual((root / "config.conf").read_text(), "original\n")
+ raise ConfigurationError("native parser rejected")
+ with self.assertRaises(ConfigurationError):
+ with transaction(root, {"config.conf": "invalid\n"}, reject, "root"):
+ self.fail("Invalid content was accepted")
+ self.assertEqual((root / "config.conf").read_text(), "original\n")
+ with self.assertRaises(RuntimeError):
+ with transaction(root, {"config.conf": "new\n", "new.conf": "new\n"}, lambda s: None, "root"):
+ self.assertEqual((root / "config.conf").read_text(), "new\n")
+ raise RuntimeError("activation failed")
+ self.assertEqual((root / "config.conf").read_text(), "original\n")
+ self.assertEqual((root / "config.conf").stat().st_mode & 0o777, 0o640)
+ self.assertFalse((root / "new.conf").exists())
+ with transaction(root, {"config.conf": "recovered\n"}, lambda s: None, "root") as receipt:
+ self.assertEqual(len(receipt["config.conf"]), 64)
+ self.assertEqual((root / "config.conf").read_text(), "recovered\n")
+
+ def test_managed_placeholder_expansion_does_not_change_application_variables(self):
+ values = params("nginx-conf", "listen {{listen_port}};\nreturn 200 '$host ${uri}';\n")
+ self.assertEqual(render(values, "nginx-conf", {"listen_port": 8080}),
+ "listen 8080;\nreturn 200 '$host ${uri}';\n")
+ with self.assertRaises(ConfigurationError):
+ render(params("nginx-conf", ""), "nginx-conf", {})
+
+ def test_results_never_echo_configuration_content(self):
+ request = dict(schema_version=1, definition_snapshot="a"*64,
+ execution_id="12345678-1234-4234-8234-123456789abc",
+ task_id=1, action="configure", parameters={"content": "private value"})
+ observed = result(request, "APPLIED", {"files": {}})
+ self.assertNotIn("parameters", observed)
+ self.assertNotIn("private value", json.dumps(observed))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_new_release.py b/tests/test_new_release.py
index 2f8790a..6a080a1 100644
--- a/tests/test_new_release.py
+++ b/tests/test_new_release.py
@@ -68,13 +68,13 @@ class NewReleaseTests(unittest.TestCase):
self.assertIn(link["port"]["http_property"], properties)
def test_shared_file_locks_match_current_release(self):
- for name in NEW + ("airflow", "celeborn", "kyuubi"):
+ for name in NEW + ("nginx", "postgresql", "airflow", "celeborn", "kyuubi"):
with self.subTest(name=name):
manifest = json.loads((ROOT / "mpacks" / name / "shared-files.json").read_text())
self.assertEqual(manifest["schema_version"], 1)
for item in manifest["files"]:
self.assertEqual(hashlib.sha256((ROOT / item["source"]).read_bytes()).hexdigest(), item["digest"])
- for name in ("nginx", "postgresql", "generic-base"):
+ for name in ("generic-base",):
self.assertEqual(json.loads((ROOT / "release.json").read_text())["packs"][name]["version"],
{"nginx": "1.0.0.4", "postgresql": "1.0.0.5", "generic-base": "1.0.0.3"}[name])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ambari.apache.org
For additional commands, e-mail: commits-help@ambari.apache.org