You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ji...@apache.org on 2024/09/07 09:45:09 UTC
(ambari) 18/22: add mpack add stack packages
This is an automated email from the ASF dual-hosted git repository.
jialiang pushed a commit to branch eee
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit c5e3ab675bb0a4fbc542af5fd05553f8c2afe513
Author: jialiang <ji...@gmail.com>
AuthorDate: Sat Sep 7 14:28:34 2024 +0800
add mpack
add stack packages
---
.../main/resources/scripts/ambari_mpack_tools.py | 298 +++++++++++++++++++
.../BIGTOP/3.2.0/properties/stack_packages.json | 208 ++++++++++++-
contrib/utils/mpack/README.md | 85 ++++++
contrib/utils/mpack/ambari_mpack_tools.py | 324 +++++++++++++++++++++
pom.xml | 59 ++--
5 files changed, 948 insertions(+), 26 deletions(-)
diff --git a/ambari-server/src/main/resources/scripts/ambari_mpack_tools.py b/ambari-server/src/main/resources/scripts/ambari_mpack_tools.py
new file mode 100644
index 0000000000..8333f2975e
--- /dev/null
+++ b/ambari-server/src/main/resources/scripts/ambari_mpack_tools.py
@@ -0,0 +1,298 @@
+import os
+import json
+import shutil
+import logging
+from xml.etree.ElementTree import Element, SubElement, tostring, parse
+from xml.dom import minidom
+import argparse
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+
+
+class XMLUtils:
+ @staticmethod
+ def parse_xml(xml_path):
+ tree = parse(xml_path)
+ return tree.getroot()
+
+ @staticmethod
+ def create_xml_element(tag, text=None, attrib={}):
+ element = Element(tag, attrib)
+ if text:
+ element.text = text
+ return element
+
+ @staticmethod
+ def prettify_xml(element):
+ return minidom.parseString(tostring(element)).toprettyxml(indent=" ")
+
+ @staticmethod
+ def write_xml_file(path, element):
+ xmlstr = XMLUtils.prettify_xml(element)
+ with open(path, "w") as f:
+ f.write(xmlstr)
+
+
+class FileUtils:
+ @staticmethod
+ def ensure_directory(path):
+ os.makedirs(path, exist_ok=True)
+
+ @staticmethod
+ def remove_directory(path):
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ @staticmethod
+ def copy_directory(src, dst):
+ shutil.copytree(src, dst)
+
+
+class Service:
+ def __init__(self, name, version):
+ self.name = name
+ self.version = version
+
+ @staticmethod
+ def get_service_version(xml_path):
+ root = XMLUtils.parse_xml(xml_path)
+ version = root.find("./services/service/version").text
+ version = version.replace("-", ".")
+ logging.info(f"Service version: {version}")
+ return version
+
+
+class Artifact:
+ def __init__(self, name, type, source_dir, service_version=None, service_name=None, stack_name=None,
+ stack_version=None):
+ self.name = name
+ self.type = type
+ self.source_dir = source_dir
+ self.service_version = service_version
+ self.service_name = service_name
+ self.stack_name = stack_name
+ self.stack_version = stack_version
+
+ def to_dict(self):
+ artifact_dict = {
+ "name": self.name,
+ "type": self.type,
+ "source_dir": self.source_dir
+ }
+ if self.service_version:
+ artifact_dict["service_version"] = self.service_version
+ if self.service_name:
+ artifact_dict["service_versions_map"] = [
+ {
+ "service_name": self.service_name,
+ "service_version": self.service_version,
+ "applicable_stacks": [
+ {
+ "stack_name": self.stack_name,
+ "stack_version": self.stack_version
+ }
+ ]
+ }
+ ]
+ return artifact_dict
+
+
+class Mpack:
+ def __init__(self, name, version, description, min_ambari_version, max_ambari_version, artifacts=None,
+ mpack_type="service", build_number=None, branch=None):
+ self.name = name
+ self.version = version
+ self.description = description
+ self.min_ambari_version = min_ambari_version
+ self.max_ambari_version = max_ambari_version
+ self.artifacts = artifacts or []
+ self.mpack_type = mpack_type
+ self.build_number = build_number
+ self.branch = branch
+
+ def to_dict(self):
+ if self.mpack_type == "service":
+ return {
+ "type": "full-release",
+ "name": self.name,
+ "version": self.version,
+ "description": self.description,
+ "prerequisites": {
+ "min_ambari_version": self.min_ambari_version,
+ "max-ambari-version": self.max_ambari_version
+ },
+ "artifacts": [artifact.to_dict() for artifact in self.artifacts]
+ }
+ else:
+ return {
+ "type": "full-release",
+ "name": self.name,
+ "version": self.version,
+ "hash": self.build_number,
+ "branch": self.branch,
+ "description": self.description,
+ "prerequisites": {
+ "min-ambari-version": f"{self.min_ambari_version}",
+ "max-ambari-version": f"{self.max_ambari_version}"
+ },
+ "artifacts": [
+ {
+ "name": "stack-definitions",
+ "type": "stack-definitions",
+ "source_dir": "stacks"
+ }
+ ]
+ }
+
+ def write_mpack_json(self, target_dir):
+ path = os.path.join(target_dir, "mpack.json")
+ with open(path, "w") as f:
+ json.dump(self.to_dict(), f, indent=2)
+
+
+class MpackBuilder:
+ def __init__(self, ambari_dir, output_directory, service_name, stack_name, stack_version, mpack_version):
+ self.ambari_dir = ambari_dir
+ self.output_directory = output_directory
+ self.service_name = service_name
+ self.stack_name = stack_name
+ self.stack_version = stack_version
+ self.mpack_version = mpack_version
+
+ def build_mpack(self, mpack_type):
+ if mpack_type == "service":
+ mpack_name = f"{self.service_name.lower()}-ambari-mpack"
+ source_dir = self.get_service_source_dir()
+ service_version = Service.get_service_version(f"{source_dir}/metainfo.xml")
+ else: # mpack_type == "stack"
+ mpack_name = "ambari-mpack"
+ source_dir = self.get_stack_source_dir()
+ service_version = None
+
+ target_dir = self.prepare_target_directory(mpack_name)
+ self.copy_source_to_target(source_dir, target_dir, mpack_type, service_version)
+ if mpack_type == "service":
+ self.create_metainfo_common(service_version, target_dir)
+ self.create_mpack(target_dir, mpack_type, service_version)
+ self.create_mpack_tarball(self.output_directory, mpack_name)
+
+ def get_service_source_dir(self):
+ return f"{self.ambari_dir}/ambari-server/src/main/resources/stacks/{self.stack_name}/{self.stack_version}/services/{self.service_name}"
+
+ def get_stack_source_dir(self):
+ return f"{self.ambari_dir}/ambari-server/src/main/resources"
+
+ def prepare_target_directory(self, mpack_name):
+ target_dir = os.path.join(self.output_directory, mpack_name)
+ FileUtils.remove_directory(target_dir)
+ FileUtils.ensure_directory(target_dir)
+ os.chdir(target_dir)
+ return target_dir
+
+ def copy_source_to_target(self, source_dir, target_dir, mpack_type, service_version):
+ if mpack_type == "service":
+ common_dir = f"{target_dir}/common-services/{self.service_name}/{service_version}"
+ FileUtils.remove_directory(common_dir)
+ FileUtils.copy_directory(source_dir, common_dir)
+ FileUtils.ensure_directory(f"addon-services/{self.service_name}/{service_version}")
+ else: # mpack_type == "stack"
+ common_dir = f"{target_dir}/common-services/"
+ stack_dir = f"{target_dir}/stacks/BIGTOP/"
+ FileUtils.copy_directory(f"{source_dir}/common-services", common_dir)
+ FileUtils.copy_directory(f"{source_dir}/stacks/BIGTOP", stack_dir)
+
+ def build_service_mpack(self):
+ self.build_mpack("service")
+
+ def build_stack_mpack(self):
+ self.build_mpack("stack")
+
+ def create_metainfo_common(self, service_version, target_dir):
+ root = XMLUtils.create_xml_element("metainfo", attrib={"schemaVersion": "2.0"})
+ services = SubElement(root, "services")
+ service = SubElement(services, "service")
+ SubElement(service, "name").text = self.service_name
+ SubElement(service, "version").text = service_version
+ SubElement(service, "extends").text = f"common-services/{self.service_name}/{service_version}"
+ path = os.path.join(target_dir, f"addon-services/{self.service_name}/{service_version}/metainfo.xml")
+ XMLUtils.write_xml_file(path, root)
+
+ def create_mpack(self, target_dir, mpack_type, service_version=None):
+ artifacts = []
+ build_number = None
+ branch = None
+ if mpack_type == "service":
+ artifact1 = Artifact(f"{self.service_name}-service-definitions", "service-definitions", "common-services",
+ service_version)
+ artifact2 = Artifact(f"{self.service_name.lower()}-addon-service-definitions",
+ "stack-addon-service-definitions", "addon-services", service_version,
+ self.service_name,
+ self.stack_name, self.stack_version)
+ artifacts = [artifact1, artifact2]
+ else:
+ build_number = "xxasdfasfdasf"
+ branch = "udh3"
+ mpack = Mpack(f"{self.service_name.lower()}-ambari-mpack", self.mpack_version,
+ f"{self.service_name} Management Pack", "2.7.5.0.0", "3.7.5.0.0", artifacts=artifacts,
+ build_number=build_number, branch=branch, mpack_type=mpack_type)
+ mpack.write_mpack_json(target_dir)
+
+ def create_mpack_tarball(self, output_directory, mpack_name):
+ os.chdir(output_directory)
+ tarball_name = f"{mpack_name}-{self.mpack_version}.tar.gz"
+ os.system(f"tar -czvf {tarball_name} {mpack_name}")
+ logging.info(f"Created mpack tarball: {tarball_name}")
+
+
+
+def setup_options():
+ parser = argparse.ArgumentParser(description='ambari Mpack Tools.')
+
+ # Add the arguments
+ parser.add_argument('-ambari-dir',
+ type=str,
+ help='The ambari prj dir')
+ parser.add_argument('-stack-name',
+ type=str,
+ help='the ambari stack u want package')
+
+ parser.add_argument('-service-name',
+ type=str,
+ help='the ambari service u want package')
+
+ parser.add_argument('-stack-version',
+ type=str,
+ help='upload components to nexus build')
+
+ parser.add_argument('-mpack-version',
+ type=str,
+ help='upload os pkgs to nexus')
+ parser.add_argument('-output-dir',
+ type=str,
+ help='upload os pkgs to nexus')
+
+ args = parser.parse_args()
+ print(f"main program params is : {args}")
+ return args
+
+
+# Usage
+if __name__ == "__main__":
+
+ args = setup_options()
+ ambari_dir = args.ambari_dir
+ stack_name = args.stack_name
+ service_name = args.service_name
+ stack_version = args.stack_version
+ mpack_version = args.mpack_version
+ output_dir = args.output_dir
+ ambari_dir=os.path.abspath(ambari_dir)
+ output_dir=os.path.abspath(output_dir)
+
+ if service_name:
+ mpack_builder = MpackBuilder(ambari_dir, output_dir, service_name, stack_name, stack_version, mpack_version)
+ mpack_builder.build_service_mpack()
+ else:
+ mpack_builder = MpackBuilder(ambari_dir, output_dir, stack_name, stack_name, stack_version, mpack_version)
+ mpack_builder.build_stack_mpack()
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
index 3025e61075..e61d57a072 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/properties/stack_packages.json
@@ -1,6 +1,110 @@
{
"BIGTOP": {
"stack-select": {
+ "CELEBORN": {
+ "CELEBORN_MASTER": {
+ "STACK-SELECT-PACKAGE": "celeborn",
+ "INSTALL": [
+ "celeborn"
+ ],
+ "PATCH": [
+ "celeborn"
+ ],
+ "STANDARD": [
+ "celeborn"
+ ]
+ },
+ "CELEBORN_WORKER": {
+ "STACK-SELECT-PACKAGE": "celeborn",
+ "INSTALL": [
+ "celeborn"
+ ],
+ "PATCH": [
+ "celeborn"
+ ],
+ "STANDARD": [
+ "celeborn"
+ ]
+ }
+ },
+ "HADOOP_ROUTER": {
+ "ROUTER": {
+ "STACK-SELECT-PACKAGE": "hadoop-hdfs-router",
+ "INSTALL": [
+ "hadoop-hdfs-router"
+ ],
+ "PATCH": [
+ "hadoop-hdfs-router"
+ ],
+ "STANDARD": [
+ "hadoop-hdfs-router"
+ ]
+ }
+ },
+ "ALLUXIO": {
+ "ALLUXIO_MASTER": {
+ "STACK-SELECT-PACKAGE": "alluxio",
+ "INSTALL": [
+ "alluxio"
+ ],
+ "PATCH": [
+ "alluxio"
+ ],
+ "STANDARD": [
+ "alluxio"
+ ]
+ },
+ "ALLUXIO_WORKER": {
+ "STACK-SELECT-PACKAGE": "alluxio",
+ "INSTALL": [
+ "alluxio"
+ ],
+ "PATCH": [
+ "alluxio"
+ ],
+ "STANDARD": [
+ "alluxio"
+ ]
+ }
+ },
+ "TRINO": {
+ "TRINO_COORDINATOR": {
+ "STACK-SELECT-PACKAGE": "trino",
+ "INSTALL": [
+ "trino"
+ ],
+ "PATCH": [
+ "trino"
+ ],
+ "STANDARD": [
+ "trino"
+ ]
+ },
+ "TRINO_WORKER": {
+ "STACK-SELECT-PACKAGE": "trino",
+ "INSTALL": [
+ "trino"
+ ],
+ "PATCH": [
+ "trino"
+ ],
+ "STANDARD": [
+ "trino"
+ ]
+ },
+ "TRINO_CLI": {
+ "STACK-SELECT-PACKAGE": "trino-cli",
+ "INSTALL": [
+ "trino-cli"
+ ],
+ "PATCH": [
+ "trino-cli"
+ ],
+ "STANDARD": [
+ "trino-cli"
+ ]
+ }
+ },
"HBASE": {
"HBASE_CLIENT": {
"STACK-SELECT-PACKAGE": "hbase-client",
@@ -215,6 +319,20 @@
]
}
},
+ "KNOX": {
+ "KNOX_GATEWAY": {
+ "STACK-SELECT-PACKAGE": "knox-server",
+ "INSTALL": [
+ "knox-server"
+ ],
+ "PATCH": [
+ "knox-server"
+ ],
+ "STANDARD": [
+ "knox-server"
+ ]
+ }
+ },
"MAPREDUCE2": {
"HISTORYSERVER": {
"STACK-SELECT-PACKAGE": "hadoop-mapreduce-historyserver",
@@ -283,6 +401,20 @@
]
}
},
+ "KYUUBI": {
+ "KYUUBI_SERVER": {
+ "STACK-SELECT-PACKAGE": "kyuubi-server",
+ "INSTALL": [
+ "kyuubi-server"
+ ],
+ "PATCH": [
+ "kyuubi-server"
+ ],
+ "STANDARD": [
+ "kyuubi-server"
+ ]
+ }
+ },
"LIVY": {
"LIVY_CLIENT": {
"STACK-SELECT-PACKAGE": "livy-client",
@@ -336,6 +468,20 @@
]
}
},
+ "KYUUBI": {
+ "KYUUBI_SERVER": {
+ "STACK-SELECT-PACKAGE": "kyuubi-server",
+ "INSTALL": [
+ "kyuubi-server"
+ ],
+ "PATCH": [
+ "kyuubi-server"
+ ],
+ "STANDARD": [
+ "kyuubi-server"
+ ]
+ }
+ },
"RANGER": {
"RANGER_ADMIN": {
"STACK-SELECT-PACKAGE": "ranger-admin",
@@ -417,11 +563,34 @@
}
},
"YARN": {
+ "APP_TIMELINE_SERVER": {
+ "STACK-SELECT-PACKAGE": "hadoop-yarn-timelineserver",
+ "INSTALL": [
+ "hadoop-yarn-timelineserver"
+ ],
+ "PATCH": [
+ "hadoop-yarn-timelineserver"
+ ],
+ "STANDARD": [
+ "hadoop-yarn-timelineserver"
+ ]
+ },
+ "TIMELINE_READER": {
+ "STACK-SELECT-PACKAGE": "hadoop-yarn-timelinereader",
+ "INSTALL": [
+ "hadoop-yarn-timelinereader"
+ ],
+ "PATCH": [
+ "hadoop-yarn-timelinereader"
+ ],
+ "STANDARD": [
+ "hadoop-yarn-timelinereader"
+ ]
+ },
"NODEMANAGER": {
"STACK-SELECT-PACKAGE": "hadoop-yarn-nodemanager",
"INSTALL": [
- "hadoop-yarn-nodemanager",
- "hadoop-client"
+ "hadoop-yarn-nodemanager"
],
"PATCH": [
"hadoop-yarn-nodemanager"
@@ -433,8 +602,7 @@
"RESOURCEMANAGER": {
"STACK-SELECT-PACKAGE": "hadoop-yarn-resourcemanager",
"INSTALL": [
- "hadoop-yarn-resourcemanager",
- "hadoop-client"
+ "hadoop-yarn-resourcemanager"
],
"PATCH": [
"hadoop-yarn-resourcemanager"
@@ -446,8 +614,7 @@
"YARN_CLIENT": {
"STACK-SELECT-PACKAGE": "hadoop-yarn-client",
"INSTALL": [
- "hadoop-yarn-client",
- "hadoop-client"
+ "hadoop-yarn-client"
],
"PATCH": [
"hadoop-yarn-client"
@@ -455,6 +622,18 @@
"STANDARD": [
"hadoop-client"
]
+ },
+ "YARN_REGISTRY_DNS": {
+ "STACK-SELECT-PACKAGE": "hadoop-yarn-registrydns",
+ "INSTALL": [
+ "hadoop-yarn-registrydns"
+ ],
+ "PATCH": [
+ "hadoop-yarn-registrydns"
+ ],
+ "STANDARD": [
+ "hadoop-yarn-registrydns"
+ ]
}
},
"ZEPPELIN": {
@@ -541,6 +720,13 @@
"component": "kafka-broker"
}
],
+ "knox": [
+ {
+ "conf_dir": "/etc/knox/conf",
+ "current_dir": "{0}/current/knox-server/conf",
+ "component": "knox-server"
+ }
+ ],
"phoenix": [
{
"conf_dir": "/etc/phoenix/conf",
@@ -579,6 +765,13 @@
"component": "spark-client"
}
],
+ "kyuubi": [
+ {
+ "conf_dir": "/etc/kyuubi/conf",
+ "current_dir": "{0}/current/kyuubi-server/conf",
+ "component": "kyuubi-server"
+ }
+ ],
"flink": [
{
"conf_dir": "/etc/flink/conf",
@@ -635,6 +828,9 @@
"kafka"
]
},
+ "KNOX": {
+ "packages": ["knox"]
+ },
"MAPREDUCE2": {
"packages": []
},
diff --git a/contrib/utils/mpack/README.md b/contrib/utils/mpack/README.md
new file mode 100644
index 0000000000..9dd1a044e6
--- /dev/null
+++ b/contrib/utils/mpack/README.md
@@ -0,0 +1,85 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--->
+# Ambari Management Pack Tool Usage Guide
+
+The Ambari Management Pack Tool is an instrumental tool designed for comprehensively packaging service code or stack code from your project into a single MPack package. This capability greatly facilitates the process of updating your Ambari services, ensuring a smooth and efficient deployment of new features or modifications.
+
+## Use Cases
+n day-to-day operations, 99% of modifications usually occur under the Ambari Stack rather than the Ambari Server itself. In such circumstances, using MPack to update only the contents of the Stack becomes highly necessary. For instance, when you are updating the management code of an Ambari service in an already existing environment, such as the HDFS service, there's no need for manual distribution or synchronization, just simply package and update with an MPack.
+For example, when you are developing a new service or modifying a service's configuration, it can be challenging to apply changes to an existing live environment. This is because Ambari writes configurations into the database at startup, making your modifications hard to implement without reinstallation. However, using MPack can resolve this issue, as the installation of an MPack will re-enter the contents of the MPack into the database, making it a powerful tool for applying updates or [...]
+
+## Preparation
+
+Before starting, make sure you have Python 3 and Ambari Server installed, and that you have the address of the Ambari project at hand.
+
+## Overview of Steps
+
+1. **Generate MPack**: Generate the needed MPack package by providing specific commands.
+2. **Stop Ambari Server**: Stop the Ambari Server before installing a new MPack.
+3. **Install MPack**: Install the generated MPack package using specified commands.
+4. **Start Ambari Server**: Restart the Ambari Server after completing the MPack installation.
+
+### Detailed Steps
+
+#### Generate MPack
+
+Use the following command to generate an MPack package. Be sure to replace the variables with your actual path and version information.
+
+```shell
+python3 ambari-server/src/main/resources/scripts/ambari_mpack_tools.py -ambari-dir=$AMBARI_DIR -stack-name=BIGTOP -stack-version=3.3.0 -mpack-version=3.3.0 -output-dir=$OUT_DIR
+```
+
+Parameter description:
+
+- `-ambari-dir`: The path to the Ambari project.
+- `-stack-name`: The stack name of the MPack being packaged, which must be consistent with the stack name in the project.
+- `-stack-version`: Under which stack version the packaged service is.
+- `-mpack-version`: The version of the MPack package being output.
+- `-output-dir`: The location where the MPack package is output.
+
+#### Stop Ambari Server
+
+Stop the Ambari Server before installing the new MPack:
+
+```shell
+ambari-server stop
+```
+
+#### Install MPack
+
+Use the following command to install the MPack, replacing the path with the location of your MPack package:
+
+```shell
+ambari-server install-mpack -s --purge --verbose --mpack=/tmp/bgtp-ambari-mpack-1.0.0.0-SNAPSHOT-bgtp-ambari-mpack.tar.gz
+```
+
+The `--purge` option will automatically delete the previous old stack definitions and then symlink the installed MPack to the corresponding location.
+
+#### Start Ambari Server
+
+After completing the steps above, restart the Ambari Server:
+
+```shell
+ambari-server start
+```
+
+## Reference Material
+
+For more detailed information, please refer to the [Ambari official documentation](https://cwiki.apache.org/confluence/display/AMBARI/Management+Packs#ManagementPacks-ManagementPackStructure).
+
+---
+Thank you for choosing the Ambari Management Pack. We hope this documentation helps you more effectively manage and update the contents of your Ambari Stack.
\ No newline at end of file
diff --git a/contrib/utils/mpack/ambari_mpack_tools.py b/contrib/utils/mpack/ambari_mpack_tools.py
new file mode 100644
index 0000000000..bef0ff6c8c
--- /dev/null
+++ b/contrib/utils/mpack/ambari_mpack_tools.py
@@ -0,0 +1,324 @@
+import os
+import json
+import shutil
+import logging
+from xml.etree.ElementTree import Element, SubElement, tostring, parse
+from xml.dom import minidom
+import argparse
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+
+
+class XMLUtils:
+ @staticmethod
+ def parse_xml(xml_path):
+ tree = parse(xml_path)
+ return tree.getroot()
+
+ @staticmethod
+ def create_xml_element(tag, text=None, attrib={}):
+ element = Element(tag, attrib)
+ if text:
+ element.text = text
+ return element
+
+ @staticmethod
+ def prettify_xml(element):
+ return minidom.parseString(tostring(element)).toprettyxml(indent=" ")
+
+ @staticmethod
+ def write_xml_file(path, element):
+ with open(path, "w") as f:
+ f.write(XMLUtils.prettify_xml(element))
+
+
+class FileUtils:
+ @staticmethod
+ def ensure_directory(path):
+ os.makedirs(path, exist_ok=True)
+
+ @staticmethod
+ def remove_directory(path):
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ @staticmethod
+ def copy_directory(src, dst):
+ shutil.copytree(src, dst)
+
+
+class Service:
+ def __init__(self, name, version):
+ self.name = name
+ self.version = version
+
+ @staticmethod
+ def get_service_version(xml_path):
+ root = XMLUtils.parse_xml(xml_path)
+ version = root.find("./services/service/version").text
+ version = version.replace("-", ".")
+ logging.info(f"Service version: {version}")
+ return version
+
+
+class Artifact:
+ def __init__(self, name, type, source_dir, service_version=None, service_name=None, stack_name=None,
+ stack_version=None):
+ self.name = name
+ self.type = type
+ self.source_dir = source_dir
+ self.service_version = service_version
+ self.service_name = service_name
+ self.stack_name = stack_name
+ self.stack_version = stack_version
+
+ def to_dict(self):
+ artifact_dict = {
+ "name": self.name,
+ "type": self.type,
+ "source_dir": self.source_dir
+ }
+ if self.service_version:
+ artifact_dict["service_version"] = self.service_version
+ if self.service_name:
+ artifact_dict["service_versions_map"] = [
+ {
+ "service_name": self.service_name,
+ "service_version": self.service_version,
+ "applicable_stacks": [
+ {
+ "stack_name": self.stack_name,
+ "stack_version": self.stack_version
+ }
+ ]
+ }
+ ]
+ return artifact_dict
+
+
+class Mpack:
+ def __init__(self, name, version, description, min_ambari_version, max_ambari_version, artifacts=None,
+ mpack_type="service", build_number=None, branch=None):
+ self.name = name
+ self.version = version
+ self.description = description
+ self.min_ambari_version = min_ambari_version
+ self.max_ambari_version = max_ambari_version
+ self.artifacts = artifacts or []
+ self.mpack_type = mpack_type
+ self.build_number = build_number
+ self.branch = branch
+
+ def to_dict(self):
+ if self.mpack_type == "service":
+ return {
+ "type": "full-release",
+ "name": self.name,
+ "version": self.version,
+ "description": self.description,
+ "prerequisites": {
+ "min_ambari_version": self.min_ambari_version,
+ "max-ambari-version": self.max_ambari_version
+ },
+ "artifacts": [artifact.to_dict() for artifact in self.artifacts]
+ }
+ else:
+ return {
+ "type": "full-release",
+ "name": self.name,
+ "version": self.version,
+ "hash": self.build_number,
+ "branch": self.branch,
+ "description": self.description,
+ "prerequisites": {
+ "min-ambari-version": f"{self.min_ambari_version}",
+ "max-ambari-version": f"{self.max_ambari_version}"
+ },
+ "artifacts": [
+ {
+ "name": "stack-definitions",
+ "type": "stack-definitions",
+ "source_dir": "stacks"
+ }
+ ]
+ }
+
+ def write_mpack_json(self, target_dir):
+ path = os.path.join(target_dir, "mpack.json")
+ with open(path, "w") as f:
+ json.dump(self.to_dict(), f, indent=2)
+
+
+class MpackBuilder:
+ def __init__(self, ambari_dir, output_directory, service_name, stack_name, stack_version, mpack_version):
+ self.ambari_dir = ambari_dir
+ self.output_directory = output_directory
+ self.service_name = service_name
+ self.stack_name = stack_name
+ self.stack_version = stack_version
+ self.mpack_version = mpack_version
+
+ def validate_paths(self):
+ stack_dir = self.get_stack_source_dir()
+ if not os.path.exists(stack_dir):
+ raise FileNotFoundError(f"The stack directory does not exist: {stack_dir}")
+ print(f"Stack directory validation passed: {stack_dir}")
+
+ if not os.path.exists(self.output_directory):
+ raise FileNotFoundError(f"The output directory does not exist: {self.output_directory}")
+ print(f"output directory validation passed: {self.output_directory}")
+
+ if self.service_name:
+ service_dir = self.get_service_source_dir()
+ if not os.path.exists(service_dir):
+ raise FileNotFoundError(f"The service directory does not exist: {service_dir}")
+ print(f"Service directory validation passed: {service_dir}")
+
+
+
+ def build_mpack(self, mpack_type):
+ if mpack_type == "service":
+ mpack_name = f"{self.service_name.lower()}-ambari-mpack"
+ source_dir = self.get_service_source_dir()
+ service_version = Service.get_service_version(f"{source_dir}/metainfo.xml")
+ else: # mpack_type == "stack"
+ mpack_name = "ambari-mpack"
+ source_dir = self.get_stack_source_dir()
+ service_version = None
+
+ target_dir = self.prepare_target_directory(mpack_name)
+ self.copy_source_to_target(source_dir, target_dir, mpack_type, service_version)
+ if mpack_type == "service":
+ self.create_metainfo_common(service_version, target_dir)
+ self.create_mpack(target_dir, mpack_type, service_version)
+ self.create_mpack_tarball(self.output_directory, mpack_name)
+
+ def get_service_source_dir(self):
+ return f"{self.ambari_dir}/ambari-server/src/main/resources/stacks/{self.stack_name}/{self.stack_version}/services/{self.service_name}"
+
+ def get_stack_source_dir(self):
+ return f"{self.ambari_dir}/ambari-server/src/main/resources"
+
+ def prepare_target_directory(self, mpack_name):
+ target_dir = os.path.join(self.output_directory, mpack_name)
+ FileUtils.remove_directory(target_dir)
+ FileUtils.ensure_directory(target_dir)
+ os.chdir(target_dir)
+ return target_dir
+
+ def copy_source_to_target(self, source_dir, target_dir, mpack_type, service_version):
+ if mpack_type == "service":
+ common_dir = f"{target_dir}/common-services/{self.service_name}/{service_version}"
+ FileUtils.remove_directory(common_dir)
+ FileUtils.copy_directory(source_dir, common_dir)
+ FileUtils.ensure_directory(f"addon-services/{self.service_name}/{service_version}")
+ else: # for stack mpack
+ common_dir = f"{target_dir}/common-services/"
+ stack_dir = f"{target_dir}/stacks/{self.stack_name}/"
+ FileUtils.copy_directory(f"{source_dir}/common-services", common_dir)
+ FileUtils.copy_directory(f"{source_dir}/stacks/{self.stack_name}", stack_dir)
+
+
+ def build(self):
+ if self.service_name:
+ mpack_builder.build_service_mpack()
+ else:
+ mpack_builder.build_stack_mpack()
+
+
+ def build_service_mpack(self):
+ self.build_mpack("service")
+
+ def build_stack_mpack(self):
+ self.build_mpack("stack")
+
+ def create_metainfo_common(self, service_version, target_dir):
+ root = XMLUtils.create_xml_element("metainfo", attrib={"schemaVersion": "2.0"})
+ services = SubElement(root, "services")
+ service = SubElement(services, "service")
+ SubElement(service, "name").text = self.service_name
+ SubElement(service, "version").text = service_version
+ SubElement(service, "extends").text = f"common-services/{self.service_name}/{service_version}"
+ path = os.path.join(target_dir, f"addon-services/{self.service_name}/{service_version}/metainfo.xml")
+ XMLUtils.write_xml_file(path, root)
+
+ def create_mpack(self, target_dir, mpack_type, service_version=None):
+ artifacts = []
+ build_number = None
+ branch = None
+ if mpack_type == "service":
+ artifact1 = Artifact(f"{self.service_name}-service-definitions", "service-definitions", "common-services",
+ service_version)
+ artifact2 = Artifact(f"{self.service_name.lower()}-addon-service-definitions",
+ "stack-addon-service-definitions", "addon-services", service_version,
+ self.service_name,
+ self.stack_name, self.stack_version)
+ artifacts = [artifact1, artifact2]
+ else:
+ build_number = "build_number_placeholder"
+ branch = "trunk"
+ mpack = Mpack(f"{self.stack_name.lower()}-ambari-mpack", self.mpack_version,
+ f"{self.stack_name} Management Pack", "2.7.5.0.0", "3.7.5.0.0", artifacts=artifacts,
+ build_number=build_number, branch=branch, mpack_type=mpack_type)
+ mpack.write_mpack_json(target_dir)
+
+ def create_mpack_tarball(self, output_directory, mpack_name):
+ os.chdir(output_directory)
+ tarball_name = f"{mpack_name}-{self.mpack_version}.tar.gz"
+ os.system(f"tar -czvf {tarball_name} {mpack_name}")
+ logging.info(f"Created mpack tarball: {tarball_name}")
+
+
+def setup_options():
+ parser = argparse.ArgumentParser(description='ambari Mpack Tools.')
+
+ # Add the arguments
+ parser.add_argument('-ambari-dir',
+ required=True,
+ type=str,
+ help='The Ambari project directory')
+ parser.add_argument('-stack-name',
+ required=True,
+ type=str,
+ help='The Ambari stack you want to package')
+
+ parser.add_argument('-service-name',
+ type=str,
+ help='The Ambari service you want to package')
+
+ parser.add_argument('-stack-version',
+ type=str,
+ help='The stack version to package the service for. It is used when packaging a service mpack '
+ 'to specify which version of the stack the service belongs to.')
+
+ parser.add_argument('-mpack-version',
+ required=True,
+ type=str,
+ help='mpack verion')
+
+ parser.add_argument('-output-dir',
+ required=True,
+ type=str,
+ help='Location to save the packaged mpack tar')
+
+ args = parser.parse_args()
+ logging.info(f"main program params is : {args}")
+ return args
+
+
+# Usage
+if __name__ == "__main__":
+
+ args = setup_options()
+ ambari_dir = args.ambari_dir
+ stack_name = args.stack_name
+ stack_version = args.stack_version
+ service_name = args.service_name
+ mpack_version = args.mpack_version
+ output_dir = args.output_dir
+ ambari_dir = os.path.abspath(ambari_dir)
+ output_dir = os.path.abspath(output_dir)
+ mpack_builder = MpackBuilder(ambari_dir, output_dir, service_name, stack_name, stack_version, mpack_version)
+ mpack_builder.validate_paths()
+ mpack_builder.build()
diff --git a/pom.xml b/pom.xml
index cad72176fd..f94c71e33a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,6 +98,17 @@
<eclipselink.version>2.6.2</eclipselink.version>
</properties>
<pluginRepositories>
+ <pluginRepository>
+ <id>aliyun-public</id>
+ <name>aliyun nexus</name>
+ <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </pluginRepository>
<pluginRepository>
<id>maven2-repository.dev.java.net</id>
<name>Java.net Repository for Maven</name>
@@ -125,34 +136,42 @@
<repositories>
<repository>
- <id>oss.sonatype.org</id>
- <name>OSS Sonatype Staging</name>
- <url>https://oss.sonatype.org/content/groups/staging</url>
+ <id>aliyun-central</id>
+ <url>https://maven.aliyun.com/repository/central</url>
</repository>
<repository>
- <id>spring-milestones</id>
- <name>Spring Milestones</name>
- <url>https://repo.spring.io/milestone</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
+ <id>aliyun-public</id>
+ <url>https://maven.aliyun.com/repository/public</url>
</repository>
<repository>
- <id>ASF Staging</id>
- <url>https://repository.apache.org/content/groups/staging/</url>
+ <id>aliyun-snapshots</id>
+ <url>https://maven.aliyun.com/repository/apache-snapshots</url>
</repository>
<repository>
- <id>ASF Snapshots</id>
- <url>https://repository.apache.org/content/repositories/snapshots/</url>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- <releases>
- <enabled>false</enabled>
- </releases>
+ <id>apache-spring-plugin</id>
+ <url>https://maven.aliyun.com/repository/spring-plugin</url>
+ </repository>
+ <repository>
+ <id>aliyun-spring</id>
+ <url>https://maven.aliyun.com/repository/spring</url>
+ </repository>
+ <repository>
+ <id>aliyun-google</id>
+ <url>https://maven.aliyun.com/repository/google</url>
+ </repository>
+ <repository>
+ <id>aliyun-gradle-plugin</id>
+ <url>https://maven.aliyun.com/repository/gradle-plugin</url>
+ </repository>
+ <repository>
+ <id>aliyun-jcenter</id>
+ <url>https://maven.aliyun.com/repository/jcenter</url>
+ </repository>
+ <repository>
+ <id>aliyun-releases</id>
+ <url>https://maven.aliyun.com/repository/releases</url>
</repository>
</repositories>
-
<build>
<pluginManagement>
<plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ambari.apache.org
For additional commands, e-mail: commits-help@ambari.apache.org