You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/01/24 07:20:17 UTC
[kylin] branch kylin4_on_cloud updated: # init && deploy kylin4 on aws (#1801)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin4_on_cloud
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin4_on_cloud by this push:
new 08ef365 # init && deploy kylin4 on aws (#1801)
08ef365 is described below
commit 08ef36539cf6285ba1724f37dfcbf070a77b373e
Author: Tengting Xu <34...@users.noreply.github.com>
AuthorDate: Mon Jan 24 15:20:08 2022 +0800
# init && deploy kylin4 on aws (#1801)
* # init && deploy kylin4 on aws
* # add license header
---
.gitignore | 9 +
README.md | 574 ++++++
__init__.py | 16 +
backup/properties/default/kylin.properties | 67 +
.../properties/templates/kylin.properties.template | 65 +
backup/scripts/prepare-ec2-env-for-kylin4.sh | 755 +++++++
backup/scripts/prepare-ec2-env-for-spark-master.sh | 587 ++++++
backup/scripts/prepare-ec2-env-for-spark-slave.sh | 477 +++++
.../scripts/prepare-ec2-env-for-static-services.sh | 622 ++++++
backup/scripts/prepare-ec2-env-for-zk.sh | 336 +++
bin/init.sh | 81 +
client/__init__.py | 16 +
client/basic.py | 133 ++
client/lightning.py | 116 ++
.../ec2-cluster-kylin4-template.yaml | 252 +++
cloudformation_templates/ec2-cluster-kylin4.yaml | 251 +++
cloudformation_templates/ec2-cluster-rds.yaml | 149 ++
.../ec2-cluster-spark-master.yaml | 219 ++
.../ec2-cluster-spark-slave-template.yaml | 210 ++
.../ec2-cluster-spark-slave.yaml | 377 ++++
.../ec2-cluster-static-services.yaml | 211 ++
cloudformation_templates/ec2-cluster-zk.yaml | 322 +++
cloudformation_templates/ec2-or-emr-vpc.yaml | 283 +++
clouds/__init__.py | 16 +
clouds/aws.py | 385 ++++
constant/__init__.py | 16 +
constant/client.py | 24 +
constant/commands.py | 49 +
constant/config.py | 65 +
constant/deployment.py | 40 +
constant/kylin_properties_params.py | 29 +
constant/path.py | 34 +
constant/server_mode.py | 24 +
constant/yaml_files.py | 43 +
constant/yaml_params.py | 92 +
deploy.py | 98 +
engine.py | 135 ++
engine_utils.py | 255 +++
images/accesskey.png | Bin 0 -> 218426 bytes
images/accesskey2.png | Bin 0 -> 47858 bytes
images/directory.png | Bin 0 -> 80810 bytes
images/granfana2.png | Bin 0 -> 111591 bytes
images/granfana3.png | Bin 0 -> 91465 bytes
images/granfanadashboard.png | Bin 0 -> 218488 bytes
images/granfanadatasource.png | Bin 0 -> 67888 bytes
images/granfanaeditpanel.png | Bin 0 -> 132924 bytes
images/granfanapanel.png | Bin 0 -> 55420 bytes
images/jars.png | Bin 0 -> 132486 bytes
images/keypair.png | Bin 0 -> 112593 bytes
images/kylinproperties.png | Bin 0 -> 23382 bytes
images/newpanel.png | Bin 0 -> 93973 bytes
images/prometheusmetrics.png | Bin 0 -> 92838 bytes
images/prometheusquery.png | Bin 0 -> 110615 bytes
images/renderedkylinproperties.png | Bin 0 -> 24685 bytes
images/scripts.png | Bin 0 -> 144342 bytes
images/sparkmetrics.png | Bin 0 -> 231429 bytes
images/staticserviceparam.png | Bin 0 -> 89601 bytes
images/structure.png | Bin 0 -> 79664 bytes
images/tars.jpg | Bin 0 -> 219568 bytes
images/tars.png | Bin 0 -> 189254 bytes
instances/__init__.py | 16 +
instances/aws_instance.py | 2163 ++++++++++++++++++++
instances/kylin_instance.py | 26 +
instances/kylin_utils.py | 39 +
kylin_configs.yaml | 306 +++
logging.ini | 47 +
logs/.keepme | 16 +
requirements.txt | 25 +
utils.py | 188 ++
69 files changed, 10259 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..87514ad
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*/__pycache__/
+*/.DS_Store
+/venv/
+.idea
+logs/*.log
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0ab0f0e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,574 @@
+# How to deploy a Kylin4 Cluster on EC2
+
+## Target
+
+1. Deploy Kylin4 on Ec2 with Spark Standalone mode.
+2. Removed the dependency of hadoop and start quickly.
+3. Support to scale worker nodes for Spark Standalone Cluster quickly and conveniently.
+4. Improve performance for query in using `Local Cache + Soft Affinity` feature (`Experimental Feature`), please check the [details](https://mp.weixin.qq.com/s/jEPvWJwSClQcMLPm64s4fQ).
+5. Support to monitor cluster status with prometheus server and granfana.
+6. Create a Kylin4 cluster on EC2 in 10 minutes.
+
+## Structure
+
+When cluster was created, services and nodes will like below:
+
+
+
+- Services are created as the number order from 1 to 4.
+- Every machine node is presented by a white box.
+- `Kylin Node` and `Spark Worker` Node can be scaled.
+- Whole cluster will has only one RDS and only one the machine node which contains `Prometheus Server` and `Hive MetaStore` service.
+
+## Quick Start
+
+1. Initialize aws account credential on local mac, please check [details](#localaws).
+
+2. Download the source code:
+
+ ```shell
+ git clone https://github.com/Kyligence/kylin-tpch.git && cd kylin-tpch && git checkout deploy-kylin-on-aws
+ ```
+
+3. Modify the `kylin-tpch/kylin_config.yml`.
+
+ 1. Set the `AWS_REGION`.
+
+ 2. Set the `IAMRole`,please check [details](#IAM).
+
+ 3. Set the `S3_URI`, please check [details](#S3).
+
+ 4. Set the `KeyName`,please check [details](#keypair).
+
+ 5. Set the `CIDR_IP`, make sure that the `CIDR_IP` match the pattern `xxx.xxx.xxx.xxx/16[|24|32]`.
+
+ > Note:
+ >
+ > 1. this `CIDR_IP` is the specified IPv4 or IPv6 CIDR address range which an inbound rule can permit instances to receive traffic from.
+ >
+ > 2. In one word, it will let your mac which ip is in the `CIDR_IP` to access instances.
+
+4. Init local env.
+
+```She
+$KYLIN_TPCH_HOME/bin/init.sh
+```
+
+> Note: Following the information into a python virtual env and get the help messages.
+
+5. Execute commands to deploy a `default` cluster.
+
+```shell
+$ python ./deploy.py --type deploy
+```
+
+After `default` cluster is ready, you will see the message `Kylin Cluster already start successfully.` in the console.
+
+6. Execute commands to list nodes of cluster.
+
+```shell
+$ python ./deploy.py --type list
+```
+
+Then you can check the `public ip` of Kylin Node.
+
+You can access `Kylin` web by `http://{kylin public ip}:7070/kylin`.
+
+7. Destroy the `default` cluster.
+
+```shell
+$ python ./deploy.py --type destroy
+```
+
+
+
+## Quick Start For Multiple Clusters
+
+> Pre-steps is same as Quick Start steps which is from 1 to 5.
+
+1. Modify the config `CLUSTER_INDEXES` for multiple cluster.
+
+ > Note:
+ >
+ > 1. `CLUSTER_INDEXES` means that cluster index is in the range of `CLUSTER_INDEXES`.
+ > 2. If user create multiple clusters, `default` cluster always be created. If `CLUSTER_INDEXES` is (1, 3), there will be 4 cluster which contains the cluster 1, 2, 3 and `default` will be created if user execute the commands.
+ > 3. Configs for multiple clusters always are same as the `default` cluster to read from `kylin-tpch/kylin_configs.yaml`
+
+2. Copy `kylin.properties.template` for expecting clusters to deploy, please check the [details](#cluster).
+
+3. Execute commands to deploy `all` clusters.
+
+ ```shell
+ python ./deploy.py --type deploy --cluster all
+ ```
+
+4. Destroy all clusters.
+
+ ```shell
+ python ./deploy.py --type destroy --cluster all
+ ```
+
+
+
+## Prerequisites
+
+### Download source code & checkout to branch of `deploy-kylin-on-aws`
+
+commands:
+
+```shell
+git clone https://github.com/Kyligence/kylin-tpch.git && cd kylin-tpch && git checkout deploy-kylin-on-aws
+```
+
+### Initiliaze an AWS Account
+
+#### I. Create an `IAM` role<a name="IAM"></a>
+
+> Note:
+>
+> `IAM` role must have the access which contains `AmazonEC2RoleforSSM` , `AmazonSSMFullAccess` and `AmazonSSMManagedInstanceCore`.
+>
+> This `IAM` Role will be used to initialize every ec2 instances which are for creating an kylin4 cluster on aws. And it will configure in `Initilize Env of Local Mac` part.
+
+#### II. Create a `User`
+
+> Note:
+>
+> The `User` who is created in aws will be used to execute some operations on EC2 instance. So the `User` must has some authorities as below.
+
+| Services | Access level | Resources | Request condition |
+| ------------------- | ------------------------------------------------------ | ------------- | ----------------- |
+| **Cloudformation** | **Limited**: List, Read,Write | All Resources | None |
+| **EC2** | **Limited**: Read | All Resources | None |
+| **IAM** | **Limited**: List, Read, Write, Permissions management | All Resources | None |
+| **RDS** | **Limited**: List, Write, Tagging | All Resources | None |
+| **S3** | **Limited**: Write, Permissions management, Tagging | All Resources | None |
+| **SNS** | **Limited**: List | All Resources | None |
+| **STS** | **Limited**: Write | All Resources | None |
+| **Systems Manager** | **Limited**: Write | All Resources | None |
+
+#### III. Login the created `User` to create key pair and Access Key<a name="keypair"></a>
+
+> Note:
+>
+> To deploy EC2 instances for Kylin4 Cluster need key pair and access key.
+
+##### 1. Create a new `Key Pair` in the `EC2 -> Key pairs`
+
+
+
+##### 2. Genrate a new `Access Key` which in the `My Security Credentials`
+
+> Note:
+>
+> Please download the generated the csv file of `Access Key` immediately. Get the `Access Key ` and `Secret Key` to initilize local mac to access aws.
+
+
+
+
+
+
+
+
+
+### Initialize a S3 direcotry<a name="S3"></a>
+
+#### I. Create a directory on S3
+
+> Note:
+>
+> This directory will store tars, scripts and jars below. And it will also be the root path of working dir for kylin4.
+
+Example: make a directory named `kylin4-aws-test` . You can also create a directory named what you like.
+
+
+
+
+
+#### **(Optional)** II. Manually Download packages & Upload them to the S3 path which suffix is `*/tar`
+
+> Note:
+>
+> This step will automatically do by tools. So you can skip this step, or you want to check packages by yourself.
+
+
+
+> Create the directory named `tar` **in the path which was created by yourself**.
+>
+> As example, the full path would be `s3://.../kylin4-aws-test/tar`.
+
+1. Download Kylin4 package by [official website](https://kylin.apache.org/download/).
+2. Download Hadoop, [version 3.2.0](https://archive.apache.org/dist/hadoop/common/hadoop-3.2.0/hadoop-3.2.0.tar.gz).
+3. Download Spark with hadoop3.2, [version 3.1.1](https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz).
+4. Download Hive, [version 2.3.9](https://archive.apache.org/dist/hive/hive-2.3.9/apache-hive-2.3.9-bin.tar.gz).
+5. Download Zookeeper, [version 3.4.9.](https://archive.apache.org/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz)
+6. Download JDK, [version 1.8_301](https://www.oracle.com/java/technologies/javase/javase8u211-later-archive-downloads.html).
+7. Download Node Exporter, [version 1.3.1.](https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz)
+8. Download Prometheus Server, [version 2.31.1](https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz).
+9. Download Kylin4 package with local cache + soft affinity feature by [public website.](https://s3.cn-north-1.amazonaws.com.cn/public.kyligence.io/kylin/tar/apache-kylin-4.0.0-bin-spark3-soft.tar.gz)
+
+
+
+> Note:
+> If you want to use Kylin4 with local cache + soft affinity feature, please download the `experimental` package above.
+
+
+
+
+
+#### (Optional) III. Upload `kylin-tpch/backup/jars/*` to the S3 Path which suffix is `*/jars`
+
+> Note:
+>
+> This step will automatically do by tools. So you can skip this step, or you want to check jars by yourself.
+
+
+
+> Create the directory named `jars` **in the path which was created by yourself**.
+>
+> As example, the full path would be `s3://.../kylin4-aws-test/jars`.
+
+Kylin4 needed extra jars
+
+- Basic jars
+ - commons-configuration-1.3.jar
+ - mysql-connector-java-5.1.40.jar
+- Local Cache + Soft Affinity feature needed jars
+ - alluxio-2.6.1-client.jar
+ - kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar
+
+
+
+#### (Optional) IV. Upload `kylin-tpch/backup/scripts/*` to the S3 Path which suffix is `*/scripts`
+
+> Note:
+>
+> This step will automatically do by tools. So you can skip this step, or you want to check scripts by yourself.
+
+
+
+> Create the directory named `scripts` **in the path which was created by yourself**.
+>
+> As example, the full path would be `s3://.../kylin4-aws-test/scripts`.
+
+Scripts:
+
+- prepare-ec2-env-for-kylin4.sh
+- prepare-ec2-env-for-spark-master.sh
+- prepare-ec2-env-for-spark-slave.sh
+- prepare-ec2-env-for-static-services.sh
+- prepare-ec2-env-for-zk.sh
+
+
+
+### Initilize Env Of Local Mac
+
+#### I. Initilize a aws account on local mac to access AWS<a name="localaws"></a>
+
+> Use `Access Key` and `Secret Key ` above to Initilize a aws account on local mac.
+
+```shell
+$ aws configure
+AWS Access Key ID : *************
+AWS Secret Access Key : *************
+Default region name : cn-northwest-1
+Default output format : json
+```
+
+> Note:
+>
+> 1. If this command got the response `-bash: aws: command not found`, please check in [Getting started with the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html).
+> 2. Region name can be in [Available Regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions).
+> 3. Default output format is `json`.
+
+#### II. Init a Python virtual env
+
+> Note:
+>
+> Make sure that your mac already has a Python which version is 3.6.6 or later.
+
+commands:
+
+```shell
+$ ./bin/init.sh
+```
+
+> Note: Follow the information after `./bin/init.sh` to activate the python virtual env.
+
+#### III. Configure the `kylin-tpch/kylin_configs.yaml`
+
+**Required parameters**:
+
+- `AWS_REGION`: Current region for EC2 instances.
+- `IAMRole`: IAM role which has the access to aws authority. This parameter will be set to created **name** of IAM role.
+- `S3_URI`: the prefix path of storing `jars/scripts/tar`. As example, this parameter will be set to `s3://.../kylin4-aws-test`.
+- `KeyName`: Security key name is a set of security credentials that you use to prove your identity when connecting to an instance. This parameter will be set to created **name** of `key pair` .
+- `CIDR_IP`: An inbound rule permits instances to receive traffic from the specified IPv4 or IPv6 CIDR address range, or from the instances associated with the specified security group.
+- `DB_IDENTIFIER`: this param should be only one in the `RDS -> Databases` . And it will be the name of created RDS database.
+- `DB_PORT`: this param will be the port of created RDS database, default is `3306`.
+- `DB_USER`: this param will be a login ID for the master user of your DB instance, default is `root`.
+- `DB_PASSWORD`: this param will be the password of `DB_USER` to access DB instance. default is `123456test`, it's strongly suggest you to change it.
+
+#### IV. Configure the `kylin.properties` in `backup/properties` directories.<a name="cluster"></a>
+
+1. The `kylin.properties` is for starting kylin instance in the cluster.
+2. Default cluster will check the `kylin.properties` in the `kylin-tpch/backup/properties/default`, and other specific cluster will check the related num directory such as `1`, `2` and `3`.
+3. User need to create new dir for the cluster num in `kylin-tpch/backup/properties`, and name it to the `${cluster num}`, such as `1`, `2` ,`3` and so on. The range of cluster num must be in `CLUSTER_INDEXES` which is configured in the `kylin-tpch/kylin_configs.yml`.
+4. Follow the `2.` step, copy the `kylin.properties.template` which is in `kylin-tpch/backup/properties/templates` to the related `${cluster num} ` directories， and rename the template to `kylin.properties`.
+5. The range of cluster nums must match the the config `CLUSTER_INDEXES`, such as `CLUSTER_INDEXES: (1, 3)` then the directories must be `1`, `2`,`3` in the `kylin-tpch/backup/properties`.
+
+
+
+6. Cusomize the config which you want to change except the config which contains `{{ }}` mark.
+
+7. If the related `kylin.properties` was filled with cluster information, a file named `.rendered` will create in the directory.
+
+
+
+7. **(Important !!!)** If you want to re-fill the `kylin.properties` for a `kylin` node in specify cluster, you need to remove the `.rendered` file and re-copy the `kylin.propertie.template` . Redo steps from `3` to `6`.
+
+## Advanced Configs
+
+> Note:
+>
+> If you want quickly to start Kylin4 Cluster on aws, then skip this part and jump to the part of [`Run`](#run) directly.
+
+### Advaced Params
+
+There are `9` modules params for tools. Introductions as below:
+
+- EC2_VPC_PARAMS: this params of module are for creating a vpc.
+
+- EC2_RDS_PARAMS: this params of module are for creating a RDS.
+
+- EC2_STATIC_SERVICES_PARAMS: this params of module are for creating a Prometheus Server and other static services.
+
+- EC2_ZOOKEEPERS_PARAMS: this params of module are for creating a Zookeeper Cluster.
+
+- EC2_SPARK_MASTER_PARAMS: this params of module are for creating a Spark Master node.
+
+- EC2_KYLIN4_PARAMS: this params of module are for creating a Kylin4.
+
+- EC2_SPARK_WORKER_PARAMS: this params of module are for creating **Spark Workers**, default is **3** spark workers for all clusters.
+
+- EC2_KYLIN4_SCALE_PARAMS: this params of module are for scaling **Kylin4 nodes**, the range of **Kylin4 nodes** is related to `KYLIN_SCALE_UP_NODES` and `KYLIN_SCALE_DOWN_NODES`.
+
+ > Note:
+ >
+ > 1. `KYLIN_SCALE_UP_NODES` is for the range of kylin nodes to scale up.
+ > 1. `KYLIN_SCALE_DOWN_NODES` is for the range of kylin nodes to scale down.
+ > 1. The range of `KYLIN_SCALE_UP_NODES` must be contain the range of `KYLIN_SCALE_DOWN_NODES`.
+ > 1. **They are effective to all clusters which is not only `default cluster` but also other cluster which index is in `${CLUSTER_INDEXES}`.**
+
+- EC2_SPARK_SCALE_SLAVE_PARAMS: this params of module are for scaling **Spark workers**, the range of **Spark Workers ** is related to `SPARK_WORKER_SCALE_UP_NODES` and `SPARK_WORKER_SCALE_DOWN_NODES`.
+
+ > Note:
+ >
+ > 1. `SPARK_WORKER_SCALE_UP_NODES` is for the range for spark workers to scale up. **It's effective to all clusters which is not only `default cluster` but also other cluster which index is in `${CLUSTER_INDEXES}`.**
+ > 1. `SPARK_WORKER_SCALE_DOWN_NODES` is for the range for spark workers to scale down. **It's effective to all clusters which is not only `default cluster` but also other cluster which index is in `${CLUSTER_INDEXES}`.**
+ > 1. The range of `SPARK_WORKER_SCALE_UP_NODES` must be contain the range of `SPARK_WORKER_SCALE_DOWN_NODES`.
+ > 1. **They are effective to all clusters which is not only `default cluster` but also other cluster which index is in `${CLUSTER_INDEXES}`.**
+
+### Customize Configs
+
+User also can customize the params in `kylin-tpch/kylin_configs.yaml` to create an expected instances. Such as **the type of instance**, **the volume size of instance** and **the volumn type of instance** and so on.
+
+1. If you want to customize configs for instances, you must modify the `EC2Mode` from `test` to `product` in the ``kylin-tpch/kylin_configs.yml`.
+2. `Ec2Mode` is only in the parms of `EC2_STATIC_SERVICES_PARAMS`, `EC2_ZOOKEEPERS_PARAMS`, `EC2_SPARK_MASTER_PARAMS`, `EC2_KYLIN4_PARAMS`, `EC2_SPARK_WORKER_PARAMS`, `EC2_KYLIN4_SCALE_PARAMS` and `EC2_SPARK_SCALE_SLAVE_PARAMS`.
+3. So instances can be customized to effect `Monitor Node`(`EC2_STATIC_SERVICES_PARAMS`), `Zookeeper Nodes`(`EC2_ZOOKEEPERS_PARAMS`), `Spark Master Node` ( `EC2_SPARK_MASTER_PARAMS`), `Kylin4 Node`( `EC2_KYLIN4_PARAMS`), `Spark workers `(`EC2_SPARK_WORKER_PARAMS`), `Kylin4 scale nodes`(`EC2_KYLIN4_SCALE_PARAMS`) and `Spark scale workers`(`EC2_SPARK_SCALE_SLAVE_PARAMS`).
+4. Now`Ec2Mode` **only effect** the related params are `Ec2InstanceTypeFor*`,`Ec2VolumeSizeFor*` and `Ec2VolumnTypeFor`* in the params modules.
+5. If you don't change `ENABLE_LOCAL_CACHE_SOFT_AFFINITY` from `"false"` to `"true"` then cluster will created normally without `Local Cache + Soft Affinity` feature!
+
+
+
+#### Example
+
+As an example in `EC2_STATIC_SERVICES_PARAMS`:
+
+- change `Ec2Mode ` from `test`to `product`
+- change `Ec2InstanceTypeForStaticServices` from `m5.2xlarge` to `m5.4xlarge`.
+- change `Ec2VolumeSizeForStaticServicesNode` from `'20'` to `'50'.`
+- change `Ec2VolumnTypeForStaticServicesNode` from `gp2` to `standard`.
+- Then create the node of static service node will be a ``m5.4xlarge` and it attach a volume which size is `50` and type is `standard`.
+
+
+
+
+
+## Monitor
+
+Current tool also support the feature of monitor for cluster.
+
+1. User can execute `python ./deploy.py --type list` to get the `public ip` of `Static Service` .
+2. User can access the `public ip` of `Static Service` and the port is `9090` to access `Prometheus` server.
+3. User can access the `public ip` of `Static Service` and the port is `3000` to access `Granfana` server.
+
+### Quick Start For Prometheus
+
+1. Input a `Metric`, user can get the available metric from the metric exploer.
+
+
+
+2. Execute a query to generate a dashboad.
+
+
+
+3. Add a new panel for a new query.
+
+
+
+More details about `PromQL` for metric exploer, please check [official website](https://prometheus.io/docs/prometheus/latest/querying/).
+
+#### Spark Metric In Prometheus
+
+> Note:
+>
+> 1. Current tool support to monitor spark metrics, more detail in [Spark Metrics](https://spark.apache.org/docs/3.1.1/monitoring.html#executor-metrics).
+> 2. Spark metrics is start with `metrics` in Prometheus server.
+>
+> 
+
+ ### Quick Start For Granfana
+
+1. Login into dashboard with default user and password which are all `admin` at the first time.
+2. Configure the connection to prometheus server.
+
+
+
+
+
+
+
+> Just config the url with syntax `http://${private ip of static service}:9090` . The `private ip of static service` can be from the `python ./deploy.py --type list` command.
+
+3. Add a new panel.
+
+
+
+4. Add a new panel or other.
+
+
+
+5. Edit & Apply the panel.
+
+
+
+More details about the usage of Granfana in [official website](https://grafana.com/docs/grafana/latest/).
+
+## Run<a name="run"></a>
+
+Command:
+
+```shell
+python ./deploy.py --type [deploy|destroy|list|scale] --scale-type [up|down] --node-type [kylin|spark_worker] [--cluster {1..6}|all|default]
+```
+
+- deploy: create cluster(s).
+
+- destroy: destroy created cluster(s).
+
+- list: list alive nodes which are with stack name, instance id, private ip and public ip.
+
+- scale: Must be used with `--scale-type` and `--node-type`.
+
+ > Note:
+ >
+ > 1. Current support to scale up/down `kylin` or `spark_worker` for specific cluster.
+ > 2. Before scale up/down `kylin` or `spark_worker` nodes, Cluster services must be ready.
+ > 2. If you want to scale a `kylin` or `spark_worker` node to a specify cluster, please add the `--cluster ${cluster num}` to specify the expected node add to the cluster `${cluster num}`.
+
+### Command of deploy
+
+- Deploy default cluster
+
+```shell
+$ python ./deploy.py --type deploy [--cluster default]
+```
+
+- Deploy a cluster with specific cluster index.
+
+```shell
+$ python ./deploy.py --type deploy --cluster ${cluster num}
+```
+
+> Note: the `${cluster num}` must be in the range of `CLUSTER_INDEXES`.
+
+- Deploy all cluster which contain default cluster and all cluster which index in the range of `CLUSTER_INDEXES`.
+
+```sHe
+$ python ./deploy.py --type deploy --cluster all
+```
+
+### Command of destroy
+
+> Note:
+>
+> Destroy all cluster will not delete vpc, rds and monitor node. So if user don't want to hold the env, please set the `ALWAYS_DESTROY_ALL` to be `'true'`.
+
+- Destroy default cluster
+
+```shell
+$ python ./deploy.py --type destroy [--cluster default]
+```
+
+- Destroy a cluster with specific cluster index.
+
+```shell
+$ python ./deploy.py --type destroy --cluster ${cluster num}
+```
+
+> Note: the `${cluster num}` must be in the range of `CLUSTER_INDEXES`.
+
+- Destroy all cluster which contain default cluster and all cluster which index in the range of `CLUSTER_INDEXES`.
+
+```shell
+$ python ./deploy.py --type destroy --cluster all
+```
+
+### Command of list
+
+- List nodes which are with **stack name**, **instance id**, **private ip** and **public ip** in **available stacks** .
+
+```shell
+$ python ./deploy.py --type list
+```
+
+### Command of scale
+
+> Note:
+>
+> 1. Scale command must be used with `--scale-type` and `--node-type`.
+> 2. If scale command not specify a cluster num, then the scaled node(kylin or spark worker) will be add to `default`cluster.
+> 3. Scale command **not support** to **scale** node (kylin or spark worker) to **all clusters** at **one time**. It means that `python ./deploy.py --type scale --scale-type up[|down] --node-type kylin[|spark_worker] --cluster all` is invalid commad.
+> 4. Scale params which are `KYLIN_SCALE_UP_NODES`, `KYLIN_SCALE_DOWN_NODES`, `SPARK_WORKER_SCALE_UP_NODES` and `SPARK_WORKER_SCALE_DOWN_NODES` effect on all cluster. So if user want to scale node for a specify cluster, then modify the scale params before **every run time.**
+> 5. **(Important!!!)** Current cluster is created with default `3` spark workers and `1` kylin node. The `3` spark workers can not be scaled down. The `1` kylin node also can not be scaled down.
+> 6. **(Important!!!)** Cluster can only scale up or down the range of nodes which is in `KYLIN_SCALE_UP_NODES`, `KYLIN_SCALE_DOWN_NODES`, `SPARK_WORKER_SCALE_UP_NODES` and `SPARK_WORKER_SCALE_DOWN_NODES` . Not the default `3` spark workers and `1` kylin node in the cluster.
+> 7. **(Important!!!)** If user don't want to create a cluster with `3` default spark workers, then user can remove the useless node module in the `Ec2InstanceOfSlave0*` of `kylin-tpch/cloudformation_templates/ec2-cluster-spark-slave.yaml`. User need to know about the syntax of cloudformation as also.
+
+- Scale up/down kylin/spark workers in default cluster
+
+```shell
+python ./deploy.py --type scale --scale-type up[|down] --node-type kylin[|spark_worker] [--cluster default]
+```
+
+- Scale up/down kylin/spark workers in a specific cluster
+
+```shell
+python ./deploy.py --type scale --scale-type up[|down] --node-type kylin[|spark_worker] --cluster ${cluster num}
+```
+
+> Note: the `${cluster num}` must be in the range of `CLUSTER_INDEXES`.
+
+
+
+## Notes
+
+1. Current tool already open the port for some services. You can access the service by `public ip` of related EC2 instance.
+ 1. `SSH`: 22
+ 2. `Granfana`: 3000
+ 3. `Prmetheus`: 9090, 9100
+ 4. `Kylin`: 7070
+ 5. `Spark`: 8080. 4040.
+2. More about cloudformation syntax, please check [aws website](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html).
+3. Current Kylin version is 4.0.0.
+4. Current Spark version is 3.1.1.
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/backup/properties/default/kylin.properties b/backup/properties/default/kylin.properties
new file mode 100644
index 0000000..f800b5d
--- /dev/null
+++ b/backup/properties/default/kylin.properties
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# Kylin server mode, valid value [all, query, job]
+kylin.server.mode=all
+kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://{{ DB_HOST }}:{{ DB_PORT }}/kylin,username=root,password={{ DB_PASSWORD }},maxActive=10,maxIdle=10
+kylin.env.zookeeper-connect-string={{ ZOOKEEPER_HOST }}
+kylin.env.hdfs-working-dir=s3a:/{{ S3_BUCKET_PATH }}/working_dir/
+## Build Engine Resource
+kylin.engine.spark-conf.spark.eventLog.dir=s3a:/{{ S3_BUCKET_PATH }}/working_dir/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=s3a:/{{ S3_BUCKET_PATH }}/working_dir/spark-history
+kylin.engine.spark-conf.spark.master=spark://{{ SPARK_MASTER }}:7077
+
+kylin.cube.cubeplanner.enabled=true
+kylin.engine.spark-conf.spark.executor.cores=3
+kylin.engine.spark-conf.spark.executor.instances=20
+kylin.engine.spark-conf.spark.executor.memory=12GB
+kylin.engine.spark-conf.spark.executor.memoryOverhead=1GB
+
+### support prometheus
+kylin.engine.spark-conf.spark.ui.prometheus.enabled=true
+kylin.engine.spark-conf.spark.executor.processTreeMetrics.enabled=true
+
+## Parquet Column Index
+kylin.engine.spark-conf.spark.hadoop.parquet.page.size=1048576
+kylin.engine.spark-conf.spark.hadoop.parquet.page.row.count.limit=100000
+kylin.engine.spark-conf.spark.hadoop.parquet.block.size=268435456
+kylin.query.spark-conf.spark.hadoop.parquet.filter.columnindex.enabled=true
+
+## Query Engine Resource
+kylin.query.spark-conf.spark.master=spark://{{ SPARK_MASTER }}:7077
+kylin.query.spark-conf.spark.driver.cores=1
+kylin.query.spark-conf.spark.driver.memory=8GB
+kylin.query.spark-conf.spark.driver.memoryOverhead=1G
+kylin.query.spark-conf.spark.executor.instances=30
+kylin.query.spark-conf.spark.executor.cores=2
+kylin.query.spark-conf.spark.executor.memory=7G
+kylin.query.spark-conf.spark.executor.memoryOverhead=1G
+kylin.query.spark-conf.spark.sql.parquet.filterPushdown=false
+
+### support prometheus
+kylin.query.spark-conf.spark.ui.prometheus.enabled=true
+kylin.query.spark-conf.spark.executor.processTreeMetrics.enabled=true
+
+## Disable canary
+kylin.canary.sparder-context-canary-enabled=false
+## Query Cache
+kylin.query.cache-enabled=true
+
+### Prepare for cluster mode
+kylin.job.scheduler.default=100
+kylin.server.self-discovery-enabled=true
\ No newline at end of file
diff --git a/backup/properties/templates/kylin.properties.template b/backup/properties/templates/kylin.properties.template
new file mode 100644
index 0000000..94dd2dd
--- /dev/null
+++ b/backup/properties/templates/kylin.properties.template
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Kylin server mode, valid value [all, query, job]
+kylin.server.mode=all
+kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://{{ DB_HOST }}:{{ DB_PORT }}/kylin,username=root,password={{ DB_PASSWORD }},maxActive=10,maxIdle=10
+kylin.env.zookeeper-connect-string={{ ZOOKEEPER_HOST }}
+kylin.env.hdfs-working-dir=s3a:/{{ S3_BUCKET_PATH }}/working_dir/
+## Build Engine Resource
+kylin.engine.spark-conf.spark.eventLog.dir=s3a:/{{ S3_BUCKET_PATH }}/working_dir/spark-history
+kylin.engine.spark-conf.spark.history.fs.logDirectory=s3a:/{{ S3_BUCKET_PATH }}/working_dir/spark-history
+kylin.engine.spark-conf.spark.master=spark://{{ SPARK_MASTER }}:7077
+
+kylin.cube.cubeplanner.enabled=true
+kylin.engine.spark-conf.spark.executor.cores=3
+kylin.engine.spark-conf.spark.executor.instances=20
+kylin.engine.spark-conf.spark.executor.memory=12GB
+kylin.engine.spark-conf.spark.executor.memoryOverhead=1GB
+
+### support prometheus
+kylin.engine.spark-conf.spark.ui.prometheus.enabled=true
+kylin.engine.spark-conf.spark.executor.processTreeMetrics.enabled=true
+
+## Parquet Column Index
+kylin.engine.spark-conf.spark.hadoop.parquet.page.size=1048576
+kylin.engine.spark-conf.spark.hadoop.parquet.page.row.count.limit=100000
+kylin.engine.spark-conf.spark.hadoop.parquet.block.size=268435456
+kylin.query.spark-conf.spark.hadoop.parquet.filter.columnindex.enabled=true
+
+## Query Engine Resource
+kylin.query.spark-conf.spark.master=spark://{{ SPARK_MASTER }}:7077
+kylin.query.spark-conf.spark.driver.cores=1
+kylin.query.spark-conf.spark.driver.memory=8GB
+kylin.query.spark-conf.spark.driver.memoryOverhead=1G
+kylin.query.spark-conf.spark.executor.instances=30
+kylin.query.spark-conf.spark.executor.cores=2
+kylin.query.spark-conf.spark.executor.memory=7G
+kylin.query.spark-conf.spark.executor.memoryOverhead=1G
+kylin.query.spark-conf.spark.sql.parquet.filterPushdown=false
+
+### support prometheus
+kylin.query.spark-conf.spark.ui.prometheus.enabled=true
+kylin.query.spark-conf.spark.executor.processTreeMetrics.enabled=true
+
+## Disable canary
+kylin.canary.sparder-context-canary-enabled=false
+## Query Cache
+kylin.query.cache-enabled=true
+
+### Prepare for cluster mode
+kylin.job.scheduler.default=100
+kylin.server.self-discovery-enabled=true
\ No newline at end of file
diff --git a/backup/scripts/prepare-ec2-env-for-kylin4.sh b/backup/scripts/prepare-ec2-env-for-kylin4.sh
new file mode 100644
index 0000000..54a3d6d
--- /dev/null
+++ b/backup/scripts/prepare-ec2-env-for-kylin4.sh
@@ -0,0 +1,755 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Note: this script is for AWS ec2 instances
+set -e
+
+# ============= Utils function ============
+function info() {
+ # shellcheck disable=SC2145
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ # shellcheck disable=SC2145
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ # shellcheck disable=SC2145
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info")
+ shift
+ # shellcheck disable=SC2068
+ info $@
+ ;;
+ "warn")
+ shift
+ # shellcheck disable=SC2068
+ warn $@
+ ;;
+ "error")
+ shift
+ # shellcheck disable=SC2068
+ error $@
+ ;;
+ *)
+ # shellcheck disable=SC2068
+ echo -e $@
+ ;;
+ esac
+}
+
+set +e
+
+function help() {
+ logging warn "Invalid input."
+ logging warn "Usage: ${BASH_SOURCE[0]}
+ --bucket-url /path/to/bucket/without/prefix
+ --region region-for-s3
+ --db-host db-host-for-kylin
+ --db-password db-password-for-kylin
+ --db-port db-port-for-kylin
+ --db-user db-user-for-kylin
+ --kylin-mode mode-for-kylin[all|query|job]
+ --local-soft whether-to-use-local-cache+soft-affinity
+ --cluster-num specify-a-cluster"
+ exit 0
+}
+
+if [[ $# -ne 18 ]]; then
+ help
+fi
+
+while [[ $# != 0 ]]; do
+ if [[ $1 == "--bucket-url" ]]; then
+ # url same as: /xxx/kylin
+ BUCKET_SUFFIX=$2
+ elif [[ $1 == "--region" ]]; then
+ CURRENT_REGION=$2
+ elif [[ $1 == "--db-host" ]]; then
+ DATABASE_HOST=$2
+ elif [[ $1 == "--db-password" ]]; then
+ DATABASE_PASSWORD=$2
+ elif [[ $1 == "--db-user" ]]; then
+ DATABASE_USER=$2
+ elif [[ $1 == '--db-port' ]]; then
+ DATABASE_PORT=$2
+ elif [[ $1 == "--local-soft" ]]; then
+ LOCAL_CACHE_SOFT_AFFINITY=$2
+ elif [[ $1 == '--cluster-num' ]]; then
+ # default value is 'default', and cluster num is from 1 to positive infinity.
+ CLUSTER_NUM=$2
+ elif [[ $1 == '--is-scaled' ]]; then
+ IS_SCALED=$2
+ else
+ help
+ fi
+ shift
+ shift
+done
+
+# =============== Env Parameters =================
+# Prepare Steps
+### Parameters for Spark and Kylin
+#### ${SPARK_VERSION:0:1} get 2 from 2.4.7
+HADOOP_VERSION=3.2.0
+SPARK_VERSION=3.1.1
+KYLIN_VERSION=4.0.0
+HIVE_VERSION=2.3.9
+
+LOCAL_CACHE_DIR=/home/ec2-user/ssd
+
+### File name
+if [[ $LOCAL_CACHE_SOFT_AFFINITY == "false" ]]; then
+ KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}.tar.gz
+ DECOMPRESSED_KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}
+else
+ KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}-soft.tar.gz
+ DECOMPRESSED_KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}-soft
+
+ # Prepared the local cache dir for local cache + soft affinity
+
+ if [[ ! -d ${LOCAL_CACHE_DIR} ]]; then
+ sudo mkdir -p ${LOCAL_CACHE_DIR}
+ sudo chmod -R 777 ${LOCAL_CACHE_DIR}
+ fi
+
+ if [[ ! -d ${LOCAL_CACHE_DIR}/alluxio-cache-driver ]]; then
+ sudo mkdir -p ${LOCAL_CACHE_DIR}/alluxio-cache-driver
+ sudo chmod -R 777 ${LOCAL_CACHE_DIR}/alluxio-cache-driver
+ fi
+
+fi
+SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION:0:3}.tgz
+HADOOP_PACKAGE=hadoop-${HADOOP_VERSION}.tar.gz
+HIVE_PACKAGE=apache-hive-${HIVE_VERSION}-bin.tar.gz
+NODE_EXPORTER_PACKAGE=node_exporter-1.3.1.linux-amd64.tar.gz
+
+### Parameter for JDK 1.8
+JDK_PACKAGE=jdk-8u301-linux-x64.tar.gz
+JDK_DECOMPRESS_NAME=jdk1.8.0_301
+
+HOME_DIR=/home/ec2-user
+
+function init_env() {
+ HADOOP_DIR=${HOME_DIR}/hadoop
+ if [[ ! -d $HADOOP_DIR ]]; then
+ mkdir ${HADOOP_DIR}
+ fi
+
+ JAVA_HOME=/usr/local/java
+ JRE_HOME=${JAVA_HOME}/jre
+ HADOOP_HOME=${HADOOP_DIR}/hadoop-${HADOOP_VERSION}
+ HIVE_HOME=${HADOOP_DIR}/hive
+
+ KYLIN_HOME=${HOME_DIR}/${DECOMPRESSED_KYLIN_PACKAGE}
+ SPARK_HOME=${HADOOP_DIR}/spark
+ OUT_LOG=${HOME_DIR}/shell.stdout
+
+ # extra prometheus env
+ NODE_EXPORTER_HOME=/home/ec2-user/node_exporter
+
+ cat <<EOF >>~/.bash_profile
+## Set env variables
+### jdk env
+export JAVA_HOME=${JAVA_HOME}
+export JRE_HOME=${JRE_HOME}
+export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
+
+#### hadoop env
+export HADOOP_HOME=${HADOOP_HOME}
+
+### hive env
+export HIVE_HOME=${HIVE_HOME}
+
+### prometheus related env
+export NODE_EXPORTER_HOME=${NODE_EXPORTER_HOME}
+
+### export all path
+export PATH=$HIVE_HOME/bin:$HIVE_HOME/conf:${HADOOP_HOME}/bin:${JAVA_HOME}/bin:$PATH
+
+### other
+export HOME_DIR=${HOME_DIR}
+export KYLIN_HOME=${KYLIN_HOME}
+export SPARK_HOME=${SPARK_HOME}
+export OUT_LOG=${OUT_LOG}
+EOF
+}
+
+if [[ ! -f ~/.inited_env ]]; then
+ logging info "Env variables not init, init it first ..."
+ init_env
+ touch ~/.inited_env
+else
+ logging warn "Env variables already inited, source it ..."
+fi
+# Init env variables
+source ~/.bash_profile
+
+exec 2>>${OUT_LOG}
+set -o pipefail
+# ================ Main Functions ======================
+PATH_TO_BUCKET=s3:/${BUCKET_SUFFIX}
+CONFIG_PATH_TO_BUCKET=s3a:/${BUCKET_SUFFIX}
+
+# Main Functions and Steps
+## prepare jdk env
+function prepare_jdk() {
+ logging info "Preparing Jdk ..."
+ if [[ -f ${HOME_DIR}/.prepared_jdk ]]; then
+ logging warn "Jdk already prepared, enjoy it."
+ return
+ fi
+
+ # copy jdk from s3 bucket to ec2 instances, so user need to upload jdk package first
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${JDK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # unzip jdk: tar -C /extract/to/path -xzvf /path/to/archive.tar.gz
+ tar -zxf ${JDK_PACKAGE}
+ sudo mv ${JDK_DECOMPRESS_NAME} ${JAVA_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error "Java package was not installed well, pleas check ..."
+ exit 0
+ fi
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.prepared_jdk
+ logging info "Jdk is ready ..."
+}
+
+function init_jdk() {
+ if [[ -f ${HOME_DIR}/.inited_jdk ]]; then
+ logging warn "Jdk already inited, skip init ..."
+ return
+ fi
+ # this function is remove the unsupport tls rules in java which version of 1.8.291 and above
+ ## backup
+ cp -f $JAVA_HOME/jre/lib/security/java.security $JAVA_HOME/jre/lib/security/java.security.bak
+
+ ## modify the java.security file
+ sed -e "s/\ TLSv1,\ TLSv1.1,//g" -i $JAVA_HOME/jre/lib/security/java.security
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.inited_jdk
+}
+
+function prepare_hadoop() {
+ if [[ -f ${HOME_DIR}/.prepared_hadoop ]]; then
+ logging warn "Hadoop already prepared, skip init ..."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HADOOP_PACKAGE} ]]; then
+ logging warn "Hadoop package ${HADOOP_PACKAGE} already downloaded, skip download ..."
+ else
+ logging info "Downloading Hadoop package ${HADOOP_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HADOOP_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/hadoop-${HADOOP_VERSION} ]]; then
+ logging warn "Hadoop Package decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hadoop package ..."
+ tar -zxf ${HADOOP_PACKAGE}
+ fi
+
+ logging info "Hadoop prepared ..."
+ touch ${HOME_DIR}/.prepared_hadoop
+}
+
+function init_hadoop() {
+ # match correct region endpoints for hadoop's core-site.xml
+ if [[ ${CURRENT_REGION} == "cn-northwest-1" ]]; then
+ S3_ENDPOINT=s3.cn-northwest-1.amazonaws.com.cn
+ elif [[ ${CURRENT_REGION} == "cn-north-1" ]]; then
+ S3_ENDPOINT=s3.cn-north-1.amazonaws.com.cn
+ else
+ S3_ENDPOINT=s3.${CURRENT_REGION}.amazonaws.com
+ fi
+
+ if [[ -f ${HOME_DIR}/.inited_hadoop ]]; then
+ logging warn "Hadoop already inited, skip init ..."
+ else
+ logging info "Init hadoop config ..."
+ # replace jars for hadoop, although it don't need to start
+ cp hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/
+ cp hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/
+
+ cat <<EOF >${HOME_DIR}/hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.default.name</name>
+ <value>${CONFIG_PATH_TO_BUCKET}/working_dir</value>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>${S3_ENDPOINT}</value>
+ </property>
+</configuration>
+EOF
+
+ logging info "Moving hadoop package to ${HADOOP_HOME} ..."
+ mv ${HOME_DIR}/hadoop-${HADOOP_VERSION} ${HADOOP_HOME}
+
+ logging warn "touch ${HOME_DIR}/.inited_hadoop ... "
+ touch ${HOME_DIR}/.inited_hadoop
+ fi
+
+ logging info "Hadoop is ready ..."
+}
+
+function prepare_hive() {
+ logging info "Preparing hive ..."
+ if [[ -f ${HOME_DIR}/.prepared_hive ]]; then
+ logging warn "Hive already prepared, enjoy it."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HIVE_PACKAGE} ]]; then
+ logging warn "${HIVE_PACKAGE} already downloaded, skip download it ..."
+ else
+ logging info "Downloading ${HIVE_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HIVE_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://downloads.apache.org/hive/hive-${HIVE_VERSION}/${HIVE_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/apache-hive-${HIVE_VERSION}-bin ]]; then
+ logging warn "Hive Package already decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hive package ..."
+ tar -zxf ${HIVE_PACKAGE}
+ fi
+
+ if [[ -d ${HIVE_HOME} ]]; then
+ logging warn "Hive package already exists, skip moving ..."
+ else
+ logging info "Moving hive package ${HIVE_PACKAGE%*.*.*} to ${HIVE_HOME} ..."
+ mv ${HIVE_PACKAGE%*.*.*} ${HIVE_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error " Moving hive package failed, please check ..."
+ exit 0
+ fi
+ fi
+
+ # execute command
+ hive --version
+ if [[ $? -eq 0 ]]; then
+ logging info "Hive ${HIVE_VERSION} is ready ..."
+ else
+ logging error "Hive not prepared well, please check ..."
+ exit 0
+ fi
+
+ logging info "Hive prepared successfully ..."
+ touch ${HOME_DIR}/.prepared_hive
+}
+
+function init_hive() {
+ if [[ -f ${HOME_DIR}/.inited_hive ]]; then
+ logging warn "Hive already init, skip init ..."
+ return
+ fi
+
+ cat <<EOF >${HIVE_HOME}/conf/hive-site.xml
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>${DATABASE_PASSWORD}</value>
+ <description>password to use against metastore database</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://${DATABASE_HOST}:${DATABASE_PORT}/hive?createDatabaseIfNotExist=true&useSSL=false</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ <description>Driver class name for a JDBC metastore</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>${DATABASE_USER}</value>
+ <description>Username to use against metastore database;default is root</description>
+ </property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ <description>
+ Enforce metastore schema version consistency.
+ True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic
+ schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures
+ proper metastore schema migration. (Default)
+ False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
+ </description>
+ </property>
+</configuration>
+EOF
+
+ # resolve jars conflict
+ if [[ ! -d $HIVE_HOME/spark_jar ]]; then
+ mkdir -p $HIVE_HOME/spark_jar
+ mv $HIVE_HOME/lib/spark-* $HIVE_HOME/spark_jar/
+ mv $HIVE_HOME/lib/jackson-module-scala_2.11-2.6.5.jar $HIVE_HOME/spark_jar/
+ fi
+
+ if [[ ! -f ${HIVE_HOME}/lib/mysql-connector-java-5.1.40.jar ]]; then
+ logging warn "${HIVE_HOME}/lib/mysql-connector-java-5.1.40.jar not exist, download it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/mysql-connector-java-5.1.40.jar ${HIVE_HOME}/lib/ --region ${CURRENT_REGION}
+ fi
+
+ logging warn "touch ${HOME_DIR}/.inited_hive ..."
+ # make a tag for hive inited
+ touch ${HOME_DIR}/.inited_hive
+ logging info "Hive is ready ..."
+}
+
+function prepare_spark() {
+ if [[ -f ${HOME_DIR}/.prepared_spark ]]; then
+ logging warn "Spark already prepared, enjoy it."
+ return
+ fi
+
+ logging info "Downloading Spark-${SPARK_VERSION} ..."
+ ## download spark
+ if [[ -f ${HOME_DIR}/${SPARK_PACKAGE} ]]; then
+ logging warn "${SPARK_PACKAGE} already download, skip download it."
+ else
+ logging warn "Downloading ${SPARK_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${SPARK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/${SPARK_PACKAGE%*.*} ]]; then
+ logging warn "Spark package already decompressed, skip decompress ..."
+ else
+ logging info "Decompressing ${SPARK_PACKAGE} ..."
+ ### unzip spark tar file
+ tar -zxf ${SPARK_PACKAGE}
+ fi
+
+ if [[ -d ${SPARK_HOME} ]]; then
+ logging warn "${SPARK_HOME} already exists, skip moving ..."
+ else
+ logging info "Moving ${SPARK_PACKAGE%*.*} to ${SPARK_HOME}"
+ mv ${SPARK_PACKAGE%*.*} ${SPARK_HOME}
+ fi
+
+ logging info "Prepare Spark-${SPARK_VERSION} success."
+ touch ${HOME_DIR}/.prepared_spark
+}
+
+function init_spark() {
+ if [[ -f ${HOME_DIR}/.inited_spark ]]; then
+ logging warn "Spark already inited, enjoy it."
+ return
+ fi
+
+ # copy needed jars
+ if [[ ! -f $SPARK_HOME/jars/hadoop-aws-${HADOOP_VERSION}.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar $SPARK_HOME/jars/
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/aws-java-sdk-bundle-1.11.375.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar $SPARK_HOME/jars/
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/mysql-connector-java-5.1.40.jar ]]; then
+ cp $HIVE_HOME/lib/mysql-connector-java-5.1.40.jar $SPARK_HOME/jars/
+ fi
+
+ # hive-site.xml for spark
+ if [[ ! -f $SPARK_HOME/conf/hive-site.xml ]]; then
+ cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf/
+ fi
+
+ #Support local cache + soft affinity
+ if [[ ${LOCAL_CACHE_SOFT_AFFINITY} == "true" ]]; then
+ if [[ ! -f $SPARK_HOME/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar ]]; then
+ logging info "Downloading kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar to $SPARK_HOME/jars/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar $SPARK_HOME/jars/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/alluxio-2.6.1-client.jar ]]; then
+ logging info "Downloading alluxio-2.6.1-client.jar to $SPARK_HOME/jars/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/alluxio-2.6.1-client.jar $SPARK_HOME/jars/ --region ${CURRENT_REGION}
+ fi
+ fi
+
+ # Support prometheus metrics
+ cat <<EOF >${SPARK_HOME}/conf/metrics.properties
+*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet
+*.sink.prometheusServlet.path=/metrics/prometheus
+master.sink.prometheusServlet.path=/metrics/master/prometheus
+applications.sink.prometheusServlet.path=/metrics/applications/prometheus
+EOF
+
+ logging info "Spark inited ..."
+ touch ${HOME_DIR}/.inited_spark
+ logging info "Spark is ready ..."
+}
+
+function prepare_kylin() {
+ logging info "Preparing kylin ..."
+
+ if [[ -f ${HOME_DIR}/.prepared_kylin ]]; then
+ logging warn "Kylin already prepared ..."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${KYLIN_PACKAGE} ]]; then
+ logging warn "Kylin package already downloaded, skip download it ..."
+ else
+ logging info "Kylin-${KYLIN_VERSION} downloading ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${KYLIN_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://archive.apache.org/dist/kylin/apache-kylin-${KYLIN_VERSION}/${KYLIN_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/${DECOMPRESSED_KYLIN_PACKAGE} ]]; then
+ logging warn "Kylin package already decompress, skip decompress ..."
+ else
+ logging warn "Kylin package decompressing ..."
+ ### unzip kylin tar file
+ tar -zxf ${KYLIN_PACKAGE}
+ fi
+
+ logging info "Kylin inited ..."
+ touch ${HOME_DIR}/.prepared_kylin
+ logging info "Kylin prepared ..."
+}
+
+function init_kylin() {
+ if [[ -f ${HOME_DIR}/.inited_kylin ]]; then
+ logging warn "Kylin already inited ..."
+ return
+ fi
+
+ if [[ ! -d ${KYLIN_HOME}/ext ]]; then
+ ### create dir directory for other dependency used by kylin
+ mkdir -p ${KYLIN_HOME}/ext
+ fi
+
+ if [[ ! -f $KYLIN_HOME/ext/mysql-connector-java-5.1.40.jar ]]; then
+ aws s3 cp ${PATH_TO_BUCKET}/jars/mysql-connector-java-5.1.40.jar $KYLIN_HOME/ext/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f $KYLIN_HOME/ext/slf4j-log4j12-1.7.25.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar $KYLIN_HOME/ext/
+ fi
+
+ if [[ ! -f $KYLIN_HOME/ext/log4j-1.2.17.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/common/lib/log4j-1.2.17.jar $KYLIN_HOME/ext/
+ fi
+
+ #Support local cache + soft affinity
+ if [[ $LOCAL_CACHE_SOFT_AFFINITY == "true" ]]; then
+ if [[ ! -f $KYLIN_HOME/ext/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar ]]; then
+ logging info "Downloading kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar to $KYLIN_HOME/ext/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar $KYLIN_HOME/ext/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f $KYLIN_HOME/ext/alluxio-2.6.1-client.jar ]]; then
+ logging info "Downloading alluxio-2.6.1-client.jar to $KYLIN_HOME/ext/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/alluxio-2.6.1-client.jar $KYLIN_HOME/ext/ --region ${CURRENT_REGION}
+ fi
+ fi
+
+ # Overwrite kylin.properties
+ logging info "Overwrite kylin.properties from ${PATH_TO_BUCKET}/properties/${CLUSTER_NUM}/kylin.properties to ${KYLIN_HOME}/conf/kylin.properties in region ${CURRENT_REGION}."
+ aws s3 cp ${PATH_TO_BUCKET}/properties/${CLUSTER_NUM}/kylin.properties ${KYLIN_HOME}/conf/kylin.properties --region ${CURRENT_REGION}
+
+ if [[ ${LOCAL_CACHE_SOFT_AFFINITY} == "true" ]]; then
+ cat <<EOF >> ${KYLIN_HOME}/conf/kylin.properties
+kylin.query.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configuration=spark-executor-log4j.properties -Dlog4j.debug -Dkylin.hdfs.working.dir=\${kylin.env.hdfs-working-dir} -Dkylin.metadata.identifier=\${kylin.metadata.url.identifier} -Dkylin.spark.category=sparder -Dkylin.spark.identifier={{APP_ID}} -Dalluxio.user.client.cache.dir=${LOCAL_CACHE_DIR}/alluxio-cache-{{APP_ID}}-{{EXECUTOR_ID}}
+
+kylin.query.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current -Dalluxio.user.client.cache.dir=${LOCAL_CACHE_DIR}/alluxio-cache-driver
+
+kylin.query.spark-conf.spark.sql.sources.ignoreDataLocality=true
+kylin.query.spark-conf.spark.extraListeners=org.apache.kylin.softaffinity.scheduler.SoftAffinityListener
+kylin.query.spark-conf.spark.hadoop.spark.kylin.local-cache.enabled=true
+kylin.query.spark-conf.spark.kylin.soft-affinity.enabled=true
+kylin.query.spark-conf.spark.kylin.soft-affinity.replications.num=1
+kylin.query.spark-conf.spark.hadoop.io.file.buffer.size=524288
+kylin.query.spark-conf.spark.hadoop.fs.hdfs.impl=org.apache.kylin.cache.fs.kylin.KylinCacheFileSystem
+
+kylin.query.spark-conf.spark.hadoop.fs.s3.impl=org.apache.kylin.cache.fs.kylin.KylinCacheFileSystem
+kylin.query.spark-conf.spark.hadoop.fs.s3a.impl=org.apache.kylin.cache.fs.kylin.KylinCacheFileSystem
+
+kylin.query.spark-conf.spark.hadoop.fs.s3a.experimental.input.fadvise=random
+kylin.query.spark-conf.spark.hadoop.parquet.enable.summary-metadata=false
+kylin.query.spark-conf.spark.sql.parquet.mergeSchema=false
+kylin.query.spark-conf.spark.sql.hive.metastorePartitionPruning=true
+
+
+
+# LOCAL or BUFF
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.store.type=LOCAL
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.async.restore.enabled=true
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.async.write.enabled=true
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.async.write.threads=6
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.size=20GB
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.page.size=1MB
+kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.local.store.file.buckets=1000
+kylin.query.spark-conf.spark.hadoop.alluxio.user.update.file.accesstime.disabled=true
+EOF
+ fi
+
+ logging info "Kylin inited ..."
+ touch ${HOME_DIR}/.inited_kylin
+ logging info "Kylin is ready ..."
+}
+
+function after_start_kylin() {
+ KYLIN_WEB_LIB_PATH=$KYLIN_HOME/tomcat/webapps/kylin/WEB-INF/lib
+ if [[ ! -f $KYLIN_WEB_LIB_PATH/commons-collections-3.2.2.jar ]]; then
+ cp ${HIVE_HOME}/lib/commons-collections-3.2.2.jar $KYLIN_WEB_LIB_PATH/
+ fi
+
+ if [[ ! -f $KYLIN_WEB_LIB_PATH/commons-configuration-1.3.jar ]]; then
+ aws s3 cp ${PATH_TO_BUCKET}/jars/commons-configuration-1.3.jar $KYLIN_WEB_LIB_PATH/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f $KYLIN_WEB_LIB_PATH/aws-java-sdk-bundle-1.11.375.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/common/lib/aws-java-sdk-bundle-1.11.375.jar $KYLIN_WEB_LIB_PATH/
+ fi
+
+ if [[ ! -f $KYLIN_WEB_LIB_PATH/hadoop-aws-3.2.0.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/common/lib/hadoop-aws-3.2.0.jar $KYLIN_WEB_LIB_PATH/
+ fi
+}
+
+function start_kylin() {
+ ${KYLIN_HOME}/bin/kylin.sh start
+ sleep 30
+}
+
+function sample_for_kylin() {
+ if [[ ${IS_SCALED} == 'false' ]]; then
+ ${KYLIN_HOME}/bin/sample.sh
+ if [[ $? -ne 0 ]]; then
+ logging error "Sample for kylin is failed, please check ..."
+ else
+ logging info "Sample for kylin is successful, enjoy it ..."
+ fi
+ else
+ logging info "It is unnecessary to sample data in scaled mode. "
+ fi
+}
+
+function restart_kylin() {
+ ${KYLIN_HOME}/bin/kylin.sh restart
+}
+
+function prepare_node_exporter() {
+ logging info "Preparing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.prepared_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already prepared, skip prepare ... "
+ return
+ fi
+
+ if [[ ! -f ${HOME_DIR}/${NODE_EXPORTER_PACKAGE} ]]; then
+ logging info "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} not downloaded, downloading it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${NODE_EXPORTER_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ else
+ logging warn "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} already download, skip download it."
+ fi
+ touch ${HOME_DIR}/.prepared_node_exporter
+ logging info "NODE_EXPORTER prepared ..."
+}
+
+function init_node_exporter() {
+ logging info "Initializing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.inited_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already inited, skip init ... "
+ return
+ fi
+
+ if [[ ! -f ${NODE_EXPORTER_HOME} ]]; then
+ logging info "NODE_EXPORTER home ${NODE_EXPORTER_HOME} not ready, decompressing ${NODE_EXPORTER_PACKAGE} ..."
+ tar -zxf ${HOME_DIR}/${NODE_EXPORTER_PACKAGE}
+ mv ${HOME_DIR}/${NODE_EXPORTER_PACKAGE%.tar.gz} ${NODE_EXPORTER_HOME}
+ else
+ logging warn "NODE_EXPORTER home ${PROMETHEUS_PACKAGE} already ready."
+ fi
+ touch ${HOME_DIR}/.inited_node_exporter
+ logging info "NODE_EXPORTER inited ..."
+}
+
+function start_node_exporter() {
+ # NOTE: default node_exporter port 9100
+ logging info "Start node_exporter ..."
+ nohup ${NODE_EXPORTER_HOME}/node_exporter >>${NODE_EXPORTER_HOME}/node.log 2>&1 &
+}
+
+function prepare_packages() {
+ if [[ -f ${HOME_DIR}/.prepared_packages ]]; then
+ logging warn "Packages already prepared, skip prepare ..."
+ return
+ fi
+
+ prepare_jdk
+ init_jdk
+
+ # add extra monitor service
+ prepare_node_exporter
+ init_node_exporter
+ # start node_exporter quickly
+ start_node_exporter
+
+ prepare_hadoop
+ init_hadoop
+
+ prepare_hive
+ init_hive
+
+ prepare_spark
+ init_spark
+
+ prepare_kylin
+ init_kylin
+
+ touch ${HOME_DIR}/.prepared_packages
+ logging info "All need packages are ready ..."
+}
+
+function start_services_on_kylin() {
+ # special step for compatible jars, details in after_start_kylin
+ sample_for_kylin
+ start_kylin
+ after_start_kylin
+ restart_kylin
+}
+
+function main() {
+ prepare_packages
+ start_services_on_kylin
+}
+
+main
diff --git a/backup/scripts/prepare-ec2-env-for-spark-master.sh b/backup/scripts/prepare-ec2-env-for-spark-master.sh
new file mode 100644
index 0000000..496dab9
--- /dev/null
+++ b/backup/scripts/prepare-ec2-env-for-spark-master.sh
@@ -0,0 +1,587 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Note: this script is for AWS ec2 instances
+set -e
+
+# ============= Utils function ============
+function info() {
+ # shellcheck disable=SC2145
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ # shellcheck disable=SC2145
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ # shellcheck disable=SC2145
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info")
+ shift
+ # shellcheck disable=SC2068
+ info $@
+ ;;
+ "warn")
+ shift
+ # shellcheck disable=SC2068
+ warn $@
+ ;;
+ "error")
+ shift
+ # shellcheck disable=SC2068
+ error $@
+ ;;
+ *)
+ # shellcheck disable=SC2068
+ echo -e $@
+ ;;
+ esac
+}
+
+set +e
+
+function help() {
+ logging warn "Invalid input."
+ logging warn "Usage: ${BASH_SOURCE[0]}
+ --bucket-url /path/to/bucket/without/prefix
+ --region region-for-s3
+ --db-host db-host-for-hive-metadata
+ --db-password db-password-for-hive-metadata
+ --db-user db-user-for-hive-metadata
+ --db-port db-port-for-hive-metadata
+ --local-soft whether-to-use-local-cache+soft-affinity"
+ exit 0
+}
+
+if [[ $# -ne 14 ]]; then
+ help
+fi
+
+while [[ $# != 0 ]]; do
+ if [[ $1 == "--bucket-url" ]]; then
+ # url same as: /xxx/kylin
+ BUCKET_SUFFIX=$2
+ elif [[ $1 == "--region" ]]; then
+ CURRENT_REGION=$2
+ elif [[ $1 == "--db-host" ]]; then
+ DATABASE_HOST=$2
+ elif [[ $1 == "--db-password" ]]; then
+ DATABASE_PASSWORD=$2
+ elif [[ $1 == "--db-user" ]]; then
+ DATABASE_USER=$2
+ elif [[ $1 == "--db-port" ]]; then
+ DATABASE_PORT=$2
+ elif [[ $1 == "--local-soft" ]]; then
+ LOCAL_CACHE_SOFT_AFFINITY=$2
+ else
+ help
+ fi
+ shift
+ shift
+done
+
+# =============== Env Parameters =================
+# Prepare Steps
+### Parameters for Spark and Kylin
+#### ${SPARK_VERSION:0:1} get 2 from 2.4.7
+HADOOP_VERSION=3.2.0
+SPARK_VERSION=3.1.1
+HIVE_VERSION=2.3.9
+
+LOCAL_CACHE_DIR=/home/ec2-user/ssd
+
+### File name
+if [[ $LOCAL_CACHE_SOFT_AFFINITY == "true" ]]; then
+ # Prepared the local cache dir for local cache + soft affinity
+ if [[ ! -d ${LOCAL_CACHE_DIR} ]]; then
+ sudo mkdir -p ${LOCAL_CACHE_DIR}
+ sudo chmod -R 777 ${LOCAL_CACHE_DIR}
+ fi
+
+ if [[ ! -d ${LOCAL_CACHE_DIR}/alluxio-cache-driver ]]; then
+ sudo mkdir -p ${LOCAL_CACHE_DIR}/alluxio-cache-driver
+ sudo chmod -R 777 ${LOCAL_CACHE_DIR}/alluxio-cache-driver
+ fi
+
+fi
+SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION:0:3}.tgz
+HADOOP_PACKAGE=hadoop-${HADOOP_VERSION}.tar.gz
+HIVE_PACKAGE=apache-hive-${HIVE_VERSION}-bin.tar.gz
+NODE_EXPORTER_PACKAGE=node_exporter-1.3.1.linux-amd64.tar.gz
+
+### Parameter for JDK 1.8
+JDK_PACKAGE=jdk-8u301-linux-x64.tar.gz
+JDK_DECOMPRESS_NAME=jdk1.8.0_301
+
+HOME_DIR=/home/ec2-user
+
+function init_env() {
+ HADOOP_DIR=${HOME_DIR}/hadoop
+ if [[ ! -d $HADOOP_DIR ]]; then
+ mkdir ${HADOOP_DIR}
+ fi
+
+ JAVA_HOME=/usr/local/java
+ JRE_HOME=${JAVA_HOME}/jre
+ HADOOP_HOME=${HADOOP_DIR}/hadoop-${HADOOP_VERSION}
+ HIVE_HOME=${HADOOP_DIR}/hive
+
+ SPARK_HOME=${HADOOP_DIR}/spark
+ OUT_LOG=${HOME_DIR}/shell.stdout
+
+ # extra prometheus env
+ NODE_EXPORTER_HOME=/home/ec2-user/node_exporter
+
+ cat <<EOF >>~/.bash_profile
+## Set env variables
+### jdk env
+export JAVA_HOME=${JAVA_HOME}
+export JRE_HOME=${JRE_HOME}
+export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
+
+#### hadoop env
+export HADOOP_HOME=${HADOOP_HOME}
+
+### hive env
+export HIVE_HOME=${HIVE_HOME}
+
+### prometheus related env
+export NODE_EXPORTER_HOME=${NODE_EXPORTER_HOME}
+
+### export all path
+export PATH=$HIVE_HOME/bin:$HIVE_HOME/conf:${HADOOP_HOME}/bin:${JAVA_HOME}/bin:$PATH
+
+### other
+export HOME_DIR=${HOME_DIR}
+export SPARK_HOME=${SPARK_HOME}
+export OUT_LOG=${OUT_LOG}
+EOF
+}
+
+if [[ ! -f ~/.inited_env ]]; then
+ logging info "Env variables not init, init it first ..."
+ init_env
+ touch ~/.inited_env
+else
+ logging warn "Env variables already inited, source it ..."
+fi
+# Init env variables
+source ~/.bash_profile
+
+exec 2>>${OUT_LOG}
+set -o pipefail
+# ================ Main Functions ======================
+PATH_TO_BUCKET=s3:/${BUCKET_SUFFIX}
+CONFIG_PATH_TO_BUCKET=s3a:/${BUCKET_SUFFIX}
+
+# Main Functions and Steps
+## prepare jdk env
+function prepare_jdk() {
+ logging info "Preparing Jdk ..."
+ if [[ -f ${HOME_DIR}/.prepared_jdk ]]; then
+ logging warn "Jdk already prepared, enjoy it."
+ return
+ fi
+
+ # copy jdk from s3 bucket to ec2 instances, so user need to upload jdk package first
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${JDK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # unzip jdk: tar -C /extract/to/path -xzvf /path/to/archive.tar.gz
+ tar -zxf ${JDK_PACKAGE}
+ sudo mv ${JDK_DECOMPRESS_NAME} ${JAVA_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error "Java package was not installed well, pleas check ..."
+ exit 0
+ fi
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.prepared_jdk
+ logging info "Jdk is ready ..."
+}
+
+function init_jdk() {
+ if [[ -f ${HOME_DIR}/.inited_jdk ]]; then
+ logging warn "Jdk already inited, skip init ..."
+ return
+ fi
+ # this function is remove the unsupport tls rules in java which version of 1.8.291 and above
+ ## backup
+ cp -f $JAVA_HOME/jre/lib/security/java.security $JAVA_HOME/jre/lib/security/java.security.bak
+
+ ## modify the java.security file
+ sed -e "s/\ TLSv1,\ TLSv1.1,//g" -i $JAVA_HOME/jre/lib/security/java.security
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.inited_jdk
+}
+
+function prepare_hadoop() {
+ if [[ -f ${HOME_DIR}/.prepared_hadoop ]]; then
+ logging warn "Hadoop already prepared, skip init ..."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HADOOP_PACKAGE} ]]; then
+ logging warn "Hadoop package ${HADOOP_PACKAGE} already downloaded, skip download ..."
+ else
+ logging info "Downloading Hadoop package ${HADOOP_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HADOOP_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/hadoop-${HADOOP_VERSION} ]]; then
+ logging warn "Hadoop Package decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hadoop package ..."
+ tar -zxf ${HADOOP_PACKAGE}
+ fi
+
+ logging info "Hadoop prepared ..."
+ touch ${HOME_DIR}/.prepared_hadoop
+}
+
+function init_hadoop() {
+ # match correct region endpoints for hadoop's core-site.xml
+ if [[ ${CURRENT_REGION} == "cn-northwest-1" ]]; then
+ S3_ENDPOINT=s3.cn-northwest-1.amazonaws.com.cn
+ elif [[ ${CURRENT_REGION} == "cn-north-1" ]]; then
+ S3_ENDPOINT=s3.cn-north-1.amazonaws.com.cn
+ else
+ S3_ENDPOINT=s3.${CURRENT_REGION}.amazonaws.com
+ fi
+
+ if [[ -f ${HOME_DIR}/.inited_hadoop ]]; then
+ logging warn "Hadoop already inited, skip init ..."
+ else
+ logging info "Init hadoop config ..."
+ # replace jars for hadoop, although it don't need to start
+ cp hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/
+ cp hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/
+
+ cat <<EOF >${HOME_DIR}/hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.default.name</name>
+ <value>${CONFIG_PATH_TO_BUCKET}/working_dir</value>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>${S3_ENDPOINT}</value>
+ </property>
+</configuration>
+EOF
+
+ logging info "Moving hadoop package to ${HADOOP_HOME} ..."
+ mv ${HOME_DIR}/hadoop-${HADOOP_VERSION} ${HADOOP_HOME}
+
+ logging warn "touch ${HOME_DIR}/.inited_hadoop ... "
+ touch ${HOME_DIR}/.inited_hadoop
+ fi
+
+ logging info "Hadoop is ready ..."
+}
+
+function prepare_hive() {
+ logging info "Preparing hive ..."
+ if [[ -f ${HOME_DIR}/.prepared_hive ]]; then
+ logging warn "Hive already prepared, enjoy it."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HIVE_PACKAGE} ]]; then
+ logging warn "${HIVE_PACKAGE} already downloaded, skip download it ..."
+ else
+ logging info "Downloading ${HIVE_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HIVE_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://downloads.apache.org/hive/hive-${HIVE_VERSION}/${HIVE_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/apache-hive-${HIVE_VERSION}-bin ]]; then
+ logging warn "Hive Package already decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hive package ..."
+ tar -zxf ${HIVE_PACKAGE}
+ fi
+
+ if [[ -d ${HIVE_HOME} ]]; then
+ logging warn "Hive package already exists, skip moving ..."
+ else
+ logging info "Moving hive package ${HIVE_PACKAGE%*.*.*} to ${HIVE_HOME} ..."
+ mv ${HIVE_PACKAGE%*.*.*} ${HIVE_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error " Moving hive package failed, please check ..."
+ exit 0
+ fi
+ fi
+
+ # execute command
+ hive --version
+ if [[ $? -eq 0 ]]; then
+ logging info "Hive ${HIVE_VERSION} is ready ..."
+ else
+ logging error "Hive not prepared well, please check ..."
+ exit 0
+ fi
+
+ logging info "Hive prepared successfully ..."
+ touch ${HOME_DIR}/.prepared_hive
+}
+
+function init_hive() {
+ if [[ -f ${HOME_DIR}/.inited_hive ]]; then
+ logging warn "Hive already init, skip init ..."
+ return
+ fi
+
+ cat <<EOF >${HIVE_HOME}/conf/hive-site.xml
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>${DATABASE_PASSWORD}</value>
+ <description>password to use against metastore database</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://${DATABASE_HOST}:${DATABASE_PORT}/hive?createDatabaseIfNotExist=true&useSSL=false</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ <description>Driver class name for a JDBC metastore</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>${DATABASE_USER}</value>
+ <description>Username to use against metastore database;default is root</description>
+ </property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ <description>
+ Enforce metastore schema version consistency.
+ True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic
+ schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures
+ proper metastore schema migration. (Default)
+ False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
+ </description>
+ </property>
+</configuration>
+EOF
+
+ # resolve jars conflict
+ if [[ ! -d $HIVE_HOME/spark_jar ]]; then
+ mkdir -p $HIVE_HOME/spark_jar
+ mv $HIVE_HOME/lib/spark-* $HIVE_HOME/spark_jar/
+ mv $HIVE_HOME/lib/jackson-module-scala_2.11-2.6.5.jar $HIVE_HOME/spark_jar/
+ fi
+
+ if [[ ! -f ${HIVE_HOME}/lib/mysql-connector-java-5.1.40.jar ]]; then
+ logging warn "${HIVE_HOME}/lib/mysql-connector-java-5.1.40.jar not exist, download it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/mysql-connector-java-5.1.40.jar ${HIVE_HOME}/lib/ --region ${CURRENT_REGION}
+ fi
+
+ logging warn "touch ${HOME_DIR}/.inited_hive ..."
+ # make a tag for hive inited
+ touch ${HOME_DIR}/.inited_hive
+ logging info "Hive is ready ..."
+}
+
+function prepare_spark() {
+ if [[ -f ${HOME_DIR}/.prepared_spark ]]; then
+ logging warn "Spark already prepared, enjoy it."
+ return
+ fi
+
+ logging info "Downloading Spark-${SPARK_VERSION} ..."
+ ## download spark
+ if [[ -f ${HOME_DIR}/${SPARK_PACKAGE} ]]; then
+ logging warn "${SPARK_PACKAGE} already download, skip download it."
+ else
+ logging warn "Downloading ${SPARK_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${SPARK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/${SPARK_PACKAGE%*.*} ]]; then
+ logging warn "Spark package already decompressed, skip decompress ..."
+ else
+ logging info "Decompressing ${SPARK_PACKAGE} ..."
+ ### unzip spark tar file
+ tar -zxf ${SPARK_PACKAGE}
+ fi
+
+ if [[ -d ${SPARK_HOME} ]]; then
+ logging warn "${SPARK_HOME} already exists, skip moving ..."
+ else
+ logging info "Moving ${SPARK_PACKAGE%*.*} to ${SPARK_HOME}"
+ mv ${SPARK_PACKAGE%*.*} ${SPARK_HOME}
+ fi
+
+ logging info "Prepare Spark-${SPARK_VERSION} success."
+ touch ${HOME_DIR}/.prepared_spark
+}
+
+function init_spark() {
+ if [[ -f ${HOME_DIR}/.inited_spark ]]; then
+ logging warn "Spark already inited, enjoy it."
+ return
+ fi
+
+ # copy needed jars
+ if [[ ! -f $SPARK_HOME/jars/hadoop-aws-${HADOOP_VERSION}.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar $SPARK_HOME/jars/
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/aws-java-sdk-bundle-1.11.375.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar $SPARK_HOME/jars/
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/mysql-connector-java-5.1.40.jar ]]; then
+ cp $HIVE_HOME/lib/mysql-connector-java-5.1.40.jar $SPARK_HOME/jars/
+ fi
+
+ # hive-site.xml for spark
+ if [[ ! -f $SPARK_HOME/conf/hive-site.xml ]]; then
+ cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf/
+ fi
+
+ #Support local cache + soft affinity
+ if [[ $LOCAL_CACHE_SOFT_AFFINITY == "true" ]]; then
+ if [[ ! -f $SPARK_HOME/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar ]]; then
+ logging info "Downloading kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar to $SPARK_HOME/jars/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar $SPARK_HOME/jars/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/alluxio-2.6.1-client.jar ]]; then
+ logging info "Downloading alluxio-2.6.1-client.jar to $SPARK_HOME/jars/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/alluxio-2.6.1-client.jar $SPARK_HOME/jars/ --region ${CURRENT_REGION}
+ fi
+ fi
+
+ # Support prometheus metrics
+ cat <<EOF > ${SPARK_HOME}/conf/metrics.properties
+*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet
+*.sink.prometheusServlet.path=/metrics/prometheus
+master.sink.prometheusServlet.path=/metrics/master/prometheus
+applications.sink.prometheusServlet.path=/metrics/applications/prometheus
+EOF
+
+ logging info "Spark inited ..."
+ touch ${HOME_DIR}/.inited_spark
+ logging info "Spark is ready ..."
+}
+
+function start_spark_master() {
+ $SPARK_HOME/sbin/start-master.sh
+ if [[ $? -ne 0 ]]; then
+ logging error "spark start master failed, please check ..."
+ exit 0
+ fi
+ logging info "Start Spark master successfully ..."
+}
+
+function prepare_node_exporter() {
+ logging info "Preparing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.prepared_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already prepared, skip prepare ... "
+ return
+ fi
+
+ if [[ ! -f ${HOME_DIR}/${NODE_EXPORTER_PACKAGE} ]]; then
+ logging info "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} not downloaded, downloading it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${NODE_EXPORTER_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ else
+ logging warn "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} already download, skip download it."
+ fi
+ touch ${HOME_DIR}/.prepared_node_exporter
+ logging info "NODE_EXPORTER prepared ..."
+}
+
+function init_node_exporter() {
+ logging info "Initializing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.inited_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already inited, skip init ... "
+ return
+ fi
+
+ if [[ ! -f ${NODE_EXPORTER_HOME} ]]; then
+ logging info "NODE_EXPORTER home ${NODE_EXPORTER_HOME} not ready, decompressing ${NODE_EXPORTER_PACKAGE} ..."
+ tar -zxf ${HOME_DIR}/${NODE_EXPORTER_PACKAGE}
+ mv ${HOME_DIR}/${NODE_EXPORTER_PACKAGE%.tar.gz} ${NODE_EXPORTER_HOME}
+ else
+ logging warn "NODE_EXPORTER home ${PROMETHEUS_PACKAGE} already ready."
+ fi
+ touch ${HOME_DIR}/.inited_node_exporter
+ logging info "NODE_EXPORTER inited ..."
+}
+
+function start_node_exporter() {
+ # NOTE: default node_exporter port 9100
+ logging info "Start node_exporter ..."
+ nohup ${NODE_EXPORTER_HOME}/node_exporter >>${NODE_EXPORTER_HOME}/node.log 2>&1 &
+}
+
+function prepare_packages() {
+ if [[ -f ${HOME_DIR}/.prepared_packages ]]; then
+ logging warn "Packages already prepared, skip prepare ..."
+ return
+ fi
+
+ prepare_jdk
+ init_jdk
+
+ # add extra monitor service
+ prepare_node_exporter
+ init_node_exporter
+ # start node_exporter quickly
+ start_node_exporter
+
+ prepare_hadoop
+ init_hadoop
+ prepare_hive
+ init_hive
+
+ prepare_spark
+ init_spark
+
+ touch ${HOME_DIR}/.prepared_packages
+ logging info "All need packages are ready ..."
+}
+
+function start_services_on_master() {
+ start_spark_master
+}
+
+function main() {
+ prepare_packages
+ start_services_on_master
+}
+
+main
diff --git a/backup/scripts/prepare-ec2-env-for-spark-slave.sh b/backup/scripts/prepare-ec2-env-for-spark-slave.sh
new file mode 100644
index 0000000..7d871a5
--- /dev/null
+++ b/backup/scripts/prepare-ec2-env-for-spark-slave.sh
@@ -0,0 +1,477 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Note: this script is for AWS ec2 instances
+set -e
+
+# ============= Utils function ============
+function info() {
+ # shellcheck disable=SC2145
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ # shellcheck disable=SC2145
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ # shellcheck disable=SC2145
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info")
+ shift
+ # shellcheck disable=SC2068
+ info $@
+ ;;
+ "warn")
+ shift
+ # shellcheck disable=SC2068
+ warn $@
+ ;;
+ "error")
+ shift
+ # shellcheck disable=SC2068
+ error $@
+ ;;
+ *)
+ # shellcheck disable=SC2068
+ echo -e $@
+ ;;
+ esac
+}
+
+set +e
+
+function help() {
+ logging warn "Invalid input."
+ logging warn "Usage: ${BASH_SOURCE[0]}
+ --bucket-url /path/to/bucket
+ --master-host host_ip
+ --worker-number 1 (or 2 or 3 ....)
+ --region region-for-s3
+ --waiting-time time-for-start-services
+ --mode cluster-mode-is-product-or-test
+ --local-soft whether-to-use-local-cache+soft-affinity"
+ exit 0
+}
+
+if [[ $# -ne 14 ]]; then
+ help
+fi
+
+while [[ $# != 0 ]]; do
+ if [[ $1 == "--bucket-url" ]]; then
+ BUCKET_SUFFIX=$2
+ elif [[ $1 == "--master-host" ]]; then
+ MASTER_HOST=$2
+ elif [[ $1 == "--worker-number" ]]; then
+ WORKER=$2
+ elif [[ $1 == "--region" ]]; then
+ CURRENT_REGION=$2
+ elif [[ $1 == "--waiting-time" ]]; then
+ WAITING_TIME=$2
+ elif [[ $1 == "--mode" ]]; then
+ WORKER_MODE=$2
+ elif [[ $1 == "--local-soft" ]]; then
+ LOCAL_CACHE_SOFT_AFFINITY=$2
+ else
+ help
+ fi
+ shift
+ shift
+done
+
+# =============== Env Parameters =================
+# Prepare Steps
+## Parameter
+### Parameters for Spark
+#### ${SPARK_VERSION:0:1} get 2 from 2.4.7
+HADOOP_VERSION=3.2.0
+SPARK_VERSION=3.1.1
+KYLIN_VERSION=4.0.0
+
+### Parameter for JDK 1.8
+JDK_PACKAGE=jdk-8u301-linux-x64.tar.gz
+JDK_DECOMPRESS_NAME=jdk1.8.0_301
+
+LOCAL_CACHE_DIR=/home/ec2-user/ssd
+
+### File name
+if [[ $LOCAL_CACHE_SOFT_AFFINITY == "false" ]]; then
+ KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}.tar.gz
+ DECOMPRESSED_KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}
+else
+ KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}-soft.tar.gz
+ DECOMPRESSED_KYLIN_PACKAGE=apache-kylin-${KYLIN_VERSION}-bin-spark${SPARK_VERSION:0:1}-soft
+
+ # Prepared the local cache dir for local cache + soft affinity
+
+ if [[ ! -d ${LOCAL_CACHE_DIR} ]]; then
+ sudo mkdir -p ${LOCAL_CACHE_DIR}
+ sudo chmod -R 777 ${LOCAL_CACHE_DIR}
+ fi
+
+ if [[ ! -d ${LOCAL_CACHE_DIR}/alluxio-cache-driver ]]; then
+ sudo mkdir -p ${LOCAL_CACHE_DIR}/alluxio-cache-driver
+ sudo chmod -R 777 ${LOCAL_CACHE_DIR}/alluxio-cache-driver
+ fi
+fi
+
+SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION:0:3}.tgz
+HADOOP_PACKAGE=hadoop-${HADOOP_VERSION}.tar.gz
+NODE_EXPORTER_PACKAGE=node_exporter-1.3.1.linux-amd64.tar.gz
+
+HOME_DIR=/home/ec2-user
+
+function init_env() {
+ HADOOP_DIR=${HOME_DIR}/hadoop
+ if [[ ! -d $HADOOP_DIR ]]; then
+ mkdir ${HADOOP_DIR}
+ fi
+
+ JAVA_HOME=/usr/local/java
+ JRE_HOME=${JAVA_HOME}/jre
+ KYLIN_HOME=${HOME_DIR}/${DECOMPRESSED_KYLIN_PACKAGE}
+ SPARK_HOME=${HADOOP_DIR}/spark
+ OUT_LOG=${HOME_DIR}/shell.stdout
+ HADOOP_HOME=${HADOOP_DIR}/hadoop-${HADOOP_VERSION}
+
+ # extra prometheus env
+ NODE_EXPORTER_HOME=/home/ec2-user/node_exporter
+
+ cat <<EOF >>~/.bash_profile
+## Set env variables
+export JAVA_HOME=${JAVA_HOME}
+export JRE_HOME=${JRE_HOME}
+export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
+#### hadoop env
+export HADOOP_HOME=${HADOOP_HOME}
+
+### prometheus related env
+export NODE_EXPORTER_HOME=${NODE_EXPORTER_HOME}
+
+export PATH=${HADOOP_HOME}/bin:${JAVA_HOME}/bin:$PATH
+
+### other
+export HOME_DIR=${HOME_DIR}
+export KYLIN_HOME=${KYLIN_HOME}
+export SPARK_HOME=${SPARK_HOME}
+export OUT_LOG=$OUT_LOG
+EOF
+}
+
+if [[ ! -f ~/.inited_env ]]; then
+ logging info "Env variables not init, init it first ..."
+ init_env
+ touch ~/.inited_env
+else
+ logging warn "Env variables already inited, source it ..."
+fi
+
+source ~/.bash_profile
+exec 2>>${OUT_LOG}
+set -o pipefail
+# ================ Main Functions ======================
+
+PATH_TO_BUCKET=s3:/${BUCKET_SUFFIX}
+
+# Main Functions and Steps
+## prepare jdk env
+function prepare_jdk() {
+ logging info "Preparing Jdk ..."
+ if [[ -f ${HOME_DIR}/.prepared_jdk ]]; then
+ logging warn "Jdk already prepared, enjoy it."
+ return
+ fi
+
+ # copy jdk from s3 bucket to ec2 instances, so user need to upload jdk package first
+ logging info "Scp jdk package from s3: aws s3 cp ${PATH_TO_BUCKET}/tar/${JDK_PACKAGE} ${HOME_DIR} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${JDK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # unzip jdk: tar -C /extract/to/path -xzvf /path/to/archive.tar.gz
+ tar -zxf ${JDK_PACKAGE}
+ sudo mv ${JDK_DECOMPRESS_NAME} ${JAVA_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error "Java package was not installed well, pleas check ..."
+ exit 0
+ fi
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.prepared_jdk
+ logging info "Jdk is ready ..."
+}
+
+function init_jdk() {
+ if [[ -f ${HOME_DIR}/.inited_jdk ]]; then
+ logging warn "Jdk already inited, skip init ..."
+ return
+ fi
+ # this function is remove the unsupport tls rules in java which version of 1.8.291 and above
+ ## backup
+ cp -f ${JAVA_HOME}/jre/lib/security/java.security ${JAVA_HOME}/jre/lib/security/java.security.bak
+
+ ## modify the java.security file
+ sed -e "s/\ TLSv1,\ TLSv1.1,//g" -i ${JAVA_HOME}/jre/lib/security/java.security
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.inited_jdk
+}
+
+function prepare_hadoop() {
+ if [[ -f ${HOME_DIR}/.prepared_hadoop ]]; then
+ logging warn "Hadoop already prepared, skip init ..."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HADOOP_PACKAGE} ]]; then
+ logging warn "Hadoop package ${HADOOP_PACKAGE} already downloaded, skip download ..."
+ else
+ logging info "Downloading Hadoop package ${HADOOP_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HADOOP_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+# # wget cost lot time
+# wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/hadoop-${HADOOP_VERSION} ]]; then
+ logging warn "Hadoop Package decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hadoop package ..."
+ tar -zxf ${HADOOP_PACKAGE}
+ fi
+
+ logging info "Moving hadoop package to ${HADOOP_HOME} ..."
+ mv ${HOME_DIR}/hadoop-${HADOOP_VERSION} ${HADOOP_HOME}
+
+ logging info "Hadoop prepared ..."
+ touch ${HOME_DIR}/.prepared_hadoop
+}
+
+function prepare_spark() {
+ if [[ -f ${HOME_DIR}/.prepared_spark ]]; then
+ logging warn "Spark already prepared, enjoy it."
+ return
+ fi
+
+ logging info "Downloading Spark-${SPARK_VERSION} ..."
+ ## download spark
+ if [[ -f ${HOME_DIR}/${SPARK_PACKAGE} ]]; then
+ logging warn "${SPARK_PACKAGE} already download, skip download it."
+ else
+ logging warn "Downloading ${SPARK_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${SPARK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+# # wget cost lot time
+# wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/${SPARK_PACKAGE%*.*} ]]; then
+ logging warn "Spark package already decompressed, skip decompress ..."
+ else
+ logging info "Decompressing ${SPARK_PACKAGE} ..."
+ ### unzip spark tar file
+ tar -zxf ${SPARK_PACKAGE}
+ fi
+
+ if [[ -d ${SPARK_HOME} ]]; then
+ logging warn "${SPARK_HOME} already exists, skip moving ..."
+ else
+ logging info "Moving ${SPARK_PACKAGE%*.*} to ${SPARK_HOME}"
+ mv ${SPARK_PACKAGE%*.*} ${SPARK_HOME}
+ fi
+
+ logging info "Prepare Spark-${SPARK_VERSION} success."
+ touch ${HOME_DIR}/.prepared_spark
+}
+
+function init_spark() {
+ if [[ -f ${HOME_DIR}/.inited_spark ]]; then
+ logging warn "Spark already inited, enjoy it."
+ return
+ fi
+
+ # copy needed jars
+ if [[ ! -f $SPARK_HOME/jars/hadoop-aws-${HADOOP_VERSION}.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar $SPARK_HOME/jars/
+ if [[ $? -ne 0 ]]; then
+ logging error "Copy $HADOOP_HOME/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar $SPARK_HOME/jars/ failed ..."
+ exit 0
+ fi
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/aws-java-sdk-bundle-1.11.375.jar ]]; then
+ cp $HADOOP_HOME/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar $SPARK_HOME/jars/
+ if [[ $? -ne 0 ]]; then
+ logging error "Copy $HADOOP_HOME/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar $SPARK_HOME/jars/ failed ..."
+ exit 0
+ fi
+ fi
+
+ #Support local cache + soft affinity
+ if [[ $LOCAL_CACHE_SOFT_AFFINITY == "true" ]]; then
+ if [[ ! -f $SPARK_HOME/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar ]]; then
+ logging info "Downloading kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar to $SPARK_HOME/jars/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar $SPARK_HOME/jars/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f $SPARK_HOME/jars/alluxio-2.6.1-client.jar ]]; then
+ logging info "Downloading alluxio-2.6.1-client.jar to $SPARK_HOME/jars/ ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/alluxio-2.6.1-client.jar $SPARK_HOME/jars/ --region ${CURRENT_REGION}
+ fi
+ fi
+
+ # Support prometheus metrics
+ cat <<EOF > ${SPARK_HOME}/conf/metrics.properties
+*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet
+*.sink.prometheusServlet.path=/metrics/prometheus
+master.sink.prometheusServlet.path=/metrics/master/prometheus
+applications.sink.prometheusServlet.path=/metrics/applications/prometheus
+EOF
+
+ logging info "Spark inited ..."
+ touch ${HOME_DIR}/.inited_spark
+ logging info "Spark is ready ..."
+}
+
+function start_spark_worker() {
+ # TODO: fix hard code for waiting time
+ sleep ${WAITING_TIME}
+ if [[ $WORKER_MODE == 'product' ]]; then
+ # product:
+ # # ec2 instance type is m5.4xlarge which has 16 cores! Set 15 to Spark master.
+ # # Also set 60 GB memory for cluster
+ $SPARK_HOME/sbin/start-worker.sh ${MASTER_HOST}:7077 -c 15 -m 63G
+ else
+ # test: ec2 instance type is m5.2xlarget which has 8cores!
+ $SPARK_HOME/sbin/start-worker.sh ${MASTER_HOST}:7077
+ fi
+ if [[ $? -ne 0 ]]; then
+ logging error "spark start worker ${WORKER} failed, please check ..."
+ exit 0
+ fi
+ logging info "Start Spark worker ${WORKER} successfully ..."
+}
+
+function prepare_kylin() {
+ logging info "Preparing kylin ..."
+
+ if [[ -f ${HOME_DIR}/.prepared_kylin ]]; then
+ logging warn "Kylin already prepared ..."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${KYLIN_PACKAGE} ]]; then
+ logging warn "Kylin package already downloaded, skip download it ..."
+ else
+ logging info "Kylin-${KYLIN_VERSION} downloading ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${KYLIN_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+# # wget cost lot time
+# wget https://archive.apache.org/dist/kylin/apache-kylin-${KYLIN_VERSION}/${KYLIN_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/${DECOMPRESSED_KYLIN_PACKAGE} ]]; then
+ logging warn "Kylin package already decompress, skip decompress ..."
+ else
+ logging warn "Kylin package decompressing ..."
+ ### unzip kylin tar file
+ tar -zxf ${KYLIN_PACKAGE}
+ fi
+
+ logging info "Kylin inited ..."
+ touch ${HOME_DIR}/.prepared_kylin
+ logging info "Kylin prepared ..."
+}
+
+
+function prepare_node_exporter() {
+ logging info "Preparing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.prepared_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already prepared, skip prepare ... "
+ return
+ fi
+
+ if [[ ! -f ${HOME_DIR}/${NODE_EXPORTER_PACKAGE} ]]; then
+ logging info "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} not downloaded, downloading it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${NODE_EXPORTER_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ else
+ logging warn "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} already download, skip download it."
+ fi
+ touch ${HOME_DIR}/.prepared_node_exporter
+ logging info "NODE_EXPORTER prepared ..."
+}
+
+function init_node_exporter() {
+ logging info "Initializing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.inited_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already inited, skip init ... "
+ return
+ fi
+
+ if [[ ! -f ${NODE_EXPORTER_HOME} ]]; then
+ logging info "NODE_EXPORTER home ${NODE_EXPORTER_HOME} not ready, decompressing ${NODE_EXPORTER_PACKAGE} ..."
+ tar -zxf ${HOME_DIR}/${NODE_EXPORTER_PACKAGE}
+ mv ${HOME_DIR}/${NODE_EXPORTER_PACKAGE%.tar.gz} ${NODE_EXPORTER_HOME}
+ else
+ logging warn "NODE_EXPORTER home ${PROMETHEUS_PACKAGE} already ready."
+ fi
+ touch ${HOME_DIR}/.inited_node_exporter
+ logging info "NODE_EXPORTER inited ..."
+}
+
+function start_node_exporter() {
+ # NOTE: default node_exporter port 9100
+ logging info "Start node_exporter ..."
+ nohup ${NODE_EXPORTER_HOME}/node_exporter >> ${NODE_EXPORTER_HOME}/node.log 2>&1 &
+}
+
+function prepare_packages() {
+ if [[ -f ${HOME_DIR}/.prepared_packages ]]; then
+ logging warn "Packages already prepared, skip prepare ..."
+ return
+ fi
+
+ prepare_jdk
+ init_jdk
+
+ # add extra monitor service
+ prepare_node_exporter
+ init_node_exporter
+ # start node_exporter quickly
+ start_node_exporter
+
+ prepare_hadoop
+
+ prepare_spark
+ init_spark
+
+ prepare_kylin
+
+ touch ${HOME_DIR}/.prepared_packages
+ logging info "All need packages are ready ..."
+}
+
+function start_services_on_slave() {
+ start_spark_worker
+}
+
+function main() {
+ prepare_packages
+ start_services_on_slave
+}
+
+main
diff --git a/backup/scripts/prepare-ec2-env-for-static-services.sh b/backup/scripts/prepare-ec2-env-for-static-services.sh
new file mode 100644
index 0000000..02e02dc
--- /dev/null
+++ b/backup/scripts/prepare-ec2-env-for-static-services.sh
@@ -0,0 +1,622 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Note: this script is for Creating Zookeeper and DB to hold cluster
+# This Script contains services which are Zookeeper, JDK and Mysql DB
+set -e
+
+# ============= Utils function ============
+function info() {
+ # shellcheck disable=SC2145
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ # shellcheck disable=SC2145
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ # shellcheck disable=SC2145
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info")
+ shift
+ # shellcheck disable=SC2068
+ info $@
+ ;;
+ "warn")
+ shift
+ # shellcheck disable=SC2068
+ warn $@
+ ;;
+ "error")
+ shift
+ # shellcheck disable=SC2068
+ error $@
+ ;;
+ *)
+ # shellcheck disable=SC2068
+ echo -e $@
+ ;;
+ esac
+}
+
+set +e
+
+# =============== Env Parameters =================
+# Prepare Steps
+## Parameter
+EC2_DEFAULT_USER=ec2-user
+
+### Parameters for Spark and Kylin
+#### ${SPARK_VERSION:0:1} get 2 from 2.4.7
+GRAFANA_VERSION=8.2.6
+HIVE_VERSION=2.3.9
+HADOOP_VERSION=3.2.0
+
+### File name
+PROMETHEUS_PACKAGE=prometheus-2.31.1.linux-amd64.tar.gz
+NODE_EXPORTER_PACKAGE=node_exporter-1.3.1.linux-amd64.tar.gz
+HIVE_PACKAGE=apache-hive-${HIVE_VERSION}-bin.tar.gz
+HADOOP_PACKAGE=hadoop-${HADOOP_VERSION}.tar.gz
+
+### Parameter for JDK 1.8
+JDK_PACKAGE=jdk-8u301-linux-x64.tar.gz
+JDK_DECOMPRESS_NAME=jdk1.8.0_301
+
+HOME_DIR=/home/ec2-user
+
+function init_env() {
+ HADOOP_DIR=${HOME_DIR}/hadoop
+ if [[ ! -d $HADOOP_DIR ]]; then
+ mkdir ${HADOOP_DIR}
+ fi
+
+ JAVA_HOME=/usr/local/java
+ JRE_HOME=${JAVA_HOME}/jre
+ OUT_LOG=${HOME_DIR}/shell.stdout
+ HADOOP_HOME=${HADOOP_DIR}/hadoop-${HADOOP_VERSION}
+ HIVE_HOME=${HADOOP_DIR}/hive
+
+ # extra prometheus env
+ PROMETHEUS_HOME=/home/ec2-user/prometheus
+ NODE_EXPORTER_HOME=/home/ec2-user/node_exporter
+
+ cat <<EOF >>~/.bash_profile
+## Set env variables
+### jdk env
+export JAVA_HOME=${JAVA_HOME}
+export JRE_HOME=${JRE_HOME}
+export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
+
+### hadoop env
+export HADOOP_HOME=${HADOOP_HOME}
+
+### hive env
+export HIVE_HOME=${HIVE_HOME}
+
+### prometheus related env
+export PROMETHEUS_HOME=${PROMETHEUS_HOME}
+export NODE_EXPORTER_HOME=${NODE_EXPORTER_HOME}
+
+### export all path
+export PATH=$HIVE_HOME/bin:$HIVE_HOME/conf:${HADOOP_HOME}/bin:${JAVA_HOME}/bin:$PATH
+
+### other
+export HOME_DIR=${HOME_DIR}
+export OUT_LOG=$OUT_LOG
+EOF
+}
+
+if [[ ! -f ~/.inited_env ]]; then
+ logging info "Env variables not init, init it first ..."
+ init_env
+ touch ~/.inited_env
+else
+ logging warn "Env variables already inited, source it ..."
+fi
+source ~/.bash_profile
+exec 2>>${OUT_LOG}
+set -o pipefail
+# ================ Main Functions ======================
+function help() {
+ logging warn "Invalid input."
+ logging warn "Usage: ${BASH_SOURCE[0]}
+ --bucket-url /path/to/bucket/without/prefix
+ --region region-for-current-instance
+ --db-host host-for-hive-to-access-rds
+ --db-user user-for-hive-to-access-rds
+ --db-password password-for-hive-to-access-rds
+ --db-port port-for-hive-to-access-rds"
+ exit 0
+}
+
+if [[ $# -ne 12 ]]; then
+ help
+fi
+
+while [[ $# != 0 ]]; do
+ if [[ $1 == "--bucket-url" ]]; then
+ # url same as: /xxx/kylin
+ BUCKET_SUFFIX=$2
+ elif [[ $1 == "--region" ]]; then
+ CURRENT_REGION=$2
+ elif [[ $1 == "--db-host" ]]; then
+ DATABASE_HOST=$2
+ elif [[ $1 == "--db-password" ]]; then
+ DATABASE_PASSWORD=$2
+ elif [[ $1 == "--db-user" ]]; then
+ DATABASE_USER=$2
+ elif [[ $1 == "--db-port" ]]; then
+ DATABASE_PORT=$2
+ else
+ help
+ fi
+ shift
+ shift
+done
+
+PATH_TO_BUCKET=s3:/${BUCKET_SUFFIX}
+CONFIG_PATH_TO_BUCKET=s3a:/${BUCKET_SUFFIX}
+
+# Main Functions and Steps
+## prepare jdk env
+function prepare_jdk() {
+ logging info "Preparing Jdk ..."
+ if [[ -f ${HOME_DIR}/.prepared_jdk ]]; then
+ logging warn "Jdk already prepared, enjoy it."
+ return
+ fi
+
+ # copy jdk from s3 bucket to ec2 instances, so user need to upload jdk package first
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${JDK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # unzip jdk: tar -C /extract/to/path -xzvf /path/to/archive.tar.gz
+ tar -zxf ${JDK_PACKAGE}
+ sudo mv ${JDK_DECOMPRESS_NAME} ${JAVA_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error "Java package was not installed well, pleas check ..."
+ exit 0
+ fi
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.prepared_jdk
+ logging info "Jdk is ready ..."
+}
+
+function init_jdk() {
+ if [[ -f ${HOME_DIR}/.inited_jdk ]]; then
+ logging warn "Jdk already inited, skip init ..."
+ return
+ fi
+ # this function is remove the unsupport tls rules in java which version of 1.8.291 and above
+ ## backup
+ cp -f $JAVA_HOME/jre/lib/security/java.security $JAVA_HOME/jre/lib/security/java.security.bak
+
+ ## modify the java.security file
+ sed -e "s/\ TLSv1,\ TLSv1.1,//g" -i $JAVA_HOME/jre/lib/security/java.security
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.inited_jdk
+}
+
+function prepare_hadoop() {
+ if [[ -f ${HOME_DIR}/.prepared_hadoop ]]; then
+ logging warn "Hadoop already prepared, skip init ..."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HADOOP_PACKAGE} ]]; then
+ logging warn "Hadoop package ${HADOOP_PACKAGE} already downloaded, skip download ..."
+ else
+ logging info "Downloading Hadoop package ${HADOOP_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HADOOP_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/hadoop-${HADOOP_VERSION} ]]; then
+ logging warn "Hadoop Package decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hadoop package ..."
+ tar -zxf ${HADOOP_PACKAGE}
+ fi
+
+ logging info "Hadoop prepared ..."
+ touch ${HOME_DIR}/.prepared_hadoop
+}
+
+function init_hadoop() {
+ # match correct region endpoints for hadoop's core-site.xml
+ if [[ ${CURRENT_REGION} == "cn-northwest-1" ]]; then
+ S3_ENDPOINT=s3.cn-northwest-1.amazonaws.com.cn
+ elif [[ ${CURRENT_REGION} == "cn-north-1" ]]; then
+ S3_ENDPOINT=s3.cn-north-1.amazonaws.com.cn
+ else
+ S3_ENDPOINT=s3.${CURRENT_REGION}.amazonaws.com
+ fi
+
+ if [[ -f ${HOME_DIR}/.inited_hadoop ]]; then
+ logging warn "Hadoop already inited, skip init ..."
+ else
+ logging info "Init hadoop config ..."
+ # replace jars for hadoop, although it don't need to start
+ cp hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/aws-java-sdk-bundle-1.11.375.jar hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/
+ cp hadoop-${HADOOP_VERSION}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar hadoop-${HADOOP_VERSION}/share/hadoop/common/lib/
+
+ cat <<EOF >${HOME_DIR}/hadoop-${HADOOP_VERSION}/etc/hadoop/core-site.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>fs.default.name</name>
+ <value>${CONFIG_PATH_TO_BUCKET}/working_dir</value>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>${S3_ENDPOINT}</value>
+ </property>
+</configuration>
+EOF
+
+ logging info "Moving hadoop package to ${HADOOP_HOME} ..."
+ mv ${HOME_DIR}/hadoop-${HADOOP_VERSION} ${HADOOP_HOME}
+
+ logging warn "touch ${HOME_DIR}/.inited_hadoop ... "
+ touch ${HOME_DIR}/.inited_hadoop
+ fi
+
+ logging info "Hadoop is ready ..."
+}
+
+function prepare_hive() {
+ logging info "Preparing hive ..."
+ if [[ -f ${HOME_DIR}/.prepared_hive ]]; then
+ logging warn "Hive already prepared, enjoy it."
+ return
+ fi
+
+ if [[ -f ${HOME_DIR}/${HIVE_PACKAGE} ]]; then
+ logging warn "${HIVE_PACKAGE} already downloaded, skip download it ..."
+ else
+ logging info "Downloading ${HIVE_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${HIVE_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget https://downloads.apache.org/hive/hive-${HIVE_VERSION}/${HIVE_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/apache-hive-${HIVE_VERSION}-bin ]]; then
+ logging warn "Hive Package already decompressed, skip decompress ..."
+ else
+ logging info "Decompress Hive package ..."
+ tar -zxf ${HIVE_PACKAGE}
+ fi
+
+ if [[ -d ${HIVE_HOME} ]]; then
+ logging warn "Hive package already exists, skip moving ..."
+ else
+ logging info "Moving hive package ${HIVE_PACKAGE%*.*.*} to ${HIVE_HOME} ..."
+ mv ${HIVE_PACKAGE%*.*.*} ${HIVE_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error " Moving hive package failed, please check ..."
+ exit 0
+ fi
+ fi
+
+ # execute command
+ hive --version
+ if [[ $? -eq 0 ]]; then
+ logging info "Hive ${HIVE_VERSION} is ready ..."
+ else
+ logging error "Hive not prepared well, please check ..."
+ exit 0
+ fi
+
+ logging info "Hive prepared successfully ..."
+ touch ${HOME_DIR}/.prepared_hive
+}
+
+function init_hive() {
+ if [[ -f ${HOME_DIR}/.inited_hive ]]; then
+ logging warn "Hive already init, skip init ..."
+ return
+ fi
+
+ cat <<EOF >${HIVE_HOME}/conf/hive-site.xml
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>${DATABASE_PASSWORD}</value>
+ <description>password to use against metastore database</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://${DATABASE_HOST}:${DATABASE_PORT}/hive?createDatabaseIfNotExist=true&useSSL=false</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ <description>Driver class name for a JDBC metastore</description>
+ </property>
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>${DATABASE_USER}</value>
+ <description>Username to use against metastore database;default is root</description>
+ </property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ <description>
+ Enforce metastore schema version consistency.
+ True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic
+ schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures
+ proper metastore schema migration. (Default)
+ False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
+ </description>
+ </property>
+</configuration>
+EOF
+
+ # resolve jars conflict
+ if [[ ! -d $HIVE_HOME/spark_jar ]]; then
+ mkdir -p $HIVE_HOME/spark_jar
+ mv $HIVE_HOME/lib/spark-* $HIVE_HOME/spark_jar/
+ mv $HIVE_HOME/lib/jackson-module-scala_2.11-2.6.5.jar $HIVE_HOME/spark_jar/
+ fi
+
+ if [[ ! -f ${HIVE_HOME}/lib/mysql-connector-java-5.1.40.jar ]]; then
+ logging warn "${HIVE_HOME}/lib/mysql-connector-java-5.1.40.jar not exist, download it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/jars/mysql-connector-java-5.1.40.jar ${HIVE_HOME}/lib/ --region ${CURRENT_REGION}
+ fi
+
+ if [[ ! -f ${HOME_DIR}/.inited_hive_metadata ]]; then
+ bash -vx $HIVE_HOME/bin/schematool -dbType mysql -initSchema
+ if [[ $? -ne 0 ]]; then
+ logging error "Init hive metadata failed, Maybe it was already initialized, please check ..."
+ else
+ logging info "Hive metadata inited successfully ..."
+ fi
+ touch ${HOME_DIR}/.inited_hive_metadata
+ fi
+
+ if [[ ! -d ${HIVE_HOME}/logs ]]; then
+ logging info "Making dir ${HIVE_HOME}/logs"
+ mkdir -p $HIVE_HOME/logs
+ fi
+
+ logging warn "touch ${HOME_DIR}/.inited_hive ..."
+ # make a tag for hive inited
+ touch ${HOME_DIR}/.inited_hive
+ logging info "Hive is ready ..."
+}
+
+function start_hive_metastore() {
+ nohup $HIVE_HOME/bin/hive --service metastore >> $HIVE_HOME/logs/hivemetastorelog.log 2>&1 &
+ logging info "Hive was logging in $HIVE_HOME/logs, you can check ..."
+}
+
+## install mysql db with docker
+function start_docker() {
+ # check docker whether is running
+ docker_status=$(systemctl is-active docker)
+ if [[ $docker_status == "active" ]]; then
+ logging warn "Docker service is already running, don't need to start it ..."
+ else
+ logging warn "Docker service is stopped, starting it ..."
+ sudo systemctl start docker
+ docker_status=$(systemctl is-active docker)
+ if [[ $docker_status == "inactive" ]]; then
+ logging error "Start docker failed, please check."
+ exit 0
+ fi
+ logging info "Start docker success ..."
+ fi
+}
+
+function prepare_docker() {
+ logging info "Preparing docker ..."
+
+ if [[ -f ${HOME_DIR}/.prepared_docker ]]; then
+ logging warn "Docker service already installed, restart it."
+ return
+ fi
+
+ # check docker whether is installed
+ if [[ -x "$(command -v docker)" ]]; then
+ docker_version=$(sudo docker -v)
+ logging info "Docker is already installed, version is ${docker_version}"
+ else
+ logging warn "Docker was not installed, now install docker ..."
+ sudo yum install docker -y
+ fi
+
+ if [[ $(getent group docker) ]]; then
+ logging warn "docker group exists, don't need create it."
+ else
+ logging warn "docker does not exist, create it."
+ sudo groupadd docker
+ fi
+
+ if id -Gn ${EC2_DEFAULT_USER} | grep docker; then
+ logging warn "${EC2_DEFAULT_USER} already in group of docker"
+ else
+ logging info "Group of docker add user ${EC2_DEFAULT_USER}"
+ sudo usermod -aG docker $EC2_DEFAULT_USER
+ fi
+
+ touch ${HOME_DIR}/.prepared_docker
+ logging info "docker is ready ..."
+}
+
+function start_grafana() {
+ logging info "Preparing grafana ..."
+ if [[ -f ${HOME_DIR}/.prepared_grafana ]]; then
+ logging warn "Grafana service already installed, check it."
+ return
+ fi
+
+ start_docker
+
+ if [[ $(sudo docker ps -q -f name=grafana-${GRAFANA_VERSION}) ]]; then
+ logging warn "Grafana-${GRAFANA_VERSION} already running, skip this ..."
+ else
+ # default user is root !!!
+ sudo docker run -d --name grafana-${GRAFANA_VERSION} --restart=always -p 3000:3000 grafana/grafana:${GRAFANA_VERSION}
+
+ if [[ $? -ne 0 ]]; then
+ logging error "Grafana start in docker was failed, please check ..."
+ exit 0
+ fi
+ fi
+ logging warn "touch ${HOME_DIR}/.prepared_grafana"
+ touch ${HOME_DIR}/.prepared_grafana
+ logging info "Grafana is ready ..."
+}
+
+function prepare_prometheus() {
+ logging info "Preparing prometheus ..."
+ if [[ -f ${HOME_DIR}/.prepared_prometheus ]]; then
+ logging warn "Prometheus already prepared, skip prepare ... "
+ return
+ fi
+
+ if [[ ! -f ${HOME_DIR}/${PROMETHEUS_PACKAGE} ]]; then
+ logging info "Prometheus package ${PROMETHEUS_PACKAGE} not downloaded, downloading it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${PROMETHEUS_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ else
+ logging warn "Prometheus package ${PROMETHEUS_PACKAGE} already download, skip download it."
+ fi
+ touch ${HOME_DIR}/.prepared_prometheus
+ logging info "Prometheus prepared ..."
+}
+
+function init_prometheus() {
+ logging info "Initializing prometheus ..."
+ if [[ -f ${HOME_DIR}/.inited_prometheus ]]; then
+ logging warn "Prometheus already inited, skip init ... "
+ return
+ fi
+
+ if [[ ! -f ${PROMETHEUS_HOME} ]]; then
+ logging info "Prometheus home ${PROMETHEUS_HOME} not ready, decompressing ${PROMETHEUS_PACKAGE} ..."
+ tar -zxf ${HOME_DIR}/${PROMETHEUS_PACKAGE}
+ mv ${HOME_DIR}/${PROMETHEUS_PACKAGE%.tar.gz} ${PROMETHEUS_HOME}
+ else
+ logging warn "Prometheus home ${PROMETHEUS_PACKAGE} already ready."
+ fi
+
+ if [[ ! -d ${PROMETHEUS_HOME}/data ]]; then
+ logging info "Prometheus data dir not exists, creating it ..."
+ mkdir -p ${PROMETHEUS_HOME}/data
+ fi
+
+ touch ${HOME_DIR}/.inited_prometheus
+ logging info "Prometheus inited ..."
+ # NOTE: prometheus server will start after node_exporter on every node started.
+}
+
+function prepare_node_exporter() {
+ logging info "Preparing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.prepared_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already prepared, skip prepare ... "
+ return
+ fi
+
+ if [[ ! -f ${HOME_DIR}/${NODE_EXPORTER_PACKAGE} ]]; then
+ logging info "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} not downloaded, downloading it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${NODE_EXPORTER_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ else
+ logging warn "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} already download, skip download it."
+ fi
+ touch ${HOME_DIR}/.prepared_node_exporter
+ logging info "NODE_EXPORTER prepared ..."
+}
+
+function init_node_exporter() {
+ logging info "Initializing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.inited_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already inited, skip init ... "
+ return
+ fi
+
+ if [[ ! -f ${NODE_EXPORTER_HOME} ]]; then
+ logging info "NODE_EXPORTER home ${NODE_EXPORTER_HOME} not ready, decompressing ${NODE_EXPORTER_PACKAGE} ..."
+ tar -zxf ${HOME_DIR}/${NODE_EXPORTER_PACKAGE}
+ mv ${HOME_DIR}/${NODE_EXPORTER_PACKAGE%.tar.gz} ${NODE_EXPORTER_HOME}
+ else
+ logging warn "NODE_EXPORTER home ${PROMETHEUS_PACKAGE} already ready."
+ fi
+ touch ${HOME_DIR}/.inited_node_exporter
+ logging info "NODE_EXPORTER inited ..."
+}
+
+function start_node_exporter() {
+ # NOTE: default node_exporter port 9100
+ logging info "Start node_exporter ..."
+ nohup ${NODE_EXPORTER_HOME}/node_exporter >> ${NODE_EXPORTER_HOME}/node.log 2>&1 &
+}
+
+function prepare_packages() {
+ if [[ -f ${HOME_DIR}/.prepared_packages ]]; then
+ logging warn "Packages already prepared, skip prepare ..."
+ return
+ fi
+
+ prepare_jdk
+ init_jdk
+
+ # add extra monitor service
+ prepare_prometheus
+ init_prometheus
+
+ prepare_node_exporter
+ init_node_exporter
+
+ prepare_hadoop
+ init_hadoop
+
+ prepare_hive
+ init_hive
+
+ touch ${HOME_DIR}/.prepared_packages
+ logging info "All need packages are ready ..."
+}
+
+function start_services_on_other() {
+ start_hive_metastore
+
+ # start extra monitor service
+ # NOTE: prometheus server will start after all node_exporter on every node started.
+ start_node_exporter
+
+ # grafana will start at last
+ prepare_docker
+ start_grafana
+}
+
+function main() {
+ prepare_packages
+ start_services_on_other
+}
+
+main
diff --git a/backup/scripts/prepare-ec2-env-for-zk.sh b/backup/scripts/prepare-ec2-env-for-zk.sh
new file mode 100644
index 0000000..03483d4
--- /dev/null
+++ b/backup/scripts/prepare-ec2-env-for-zk.sh
@@ -0,0 +1,336 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Note: this script is for Creating Zookeeper and DB to hold cluster
+# This Script contains services which are Zookeeper, JDK and Mysql DB
+set -e
+
+# ============= Utils function ============
+function info() {
+ # shellcheck disable=SC2145
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ # shellcheck disable=SC2145
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ # shellcheck disable=SC2145
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info")
+ shift
+ # shellcheck disable=SC2068
+ info $@
+ ;;
+ "warn")
+ shift
+ # shellcheck disable=SC2068
+ warn $@
+ ;;
+ "error")
+ shift
+ # shellcheck disable=SC2068
+ error $@
+ ;;
+ *)
+ # shellcheck disable=SC2068
+ echo -e $@
+ ;;
+ esac
+}
+
+set +e
+
+# =============== Env Parameters =================
+# Prepare Steps
+## Parameter
+### Parameters for Spark and Kylin
+#### ${SPARK_VERSION:0:1} get 2 from 2.4.7
+ZOOKEEPER_VERSION=3.4.13
+
+### File name
+ZOOKEEPER_PACKAGE=zookeeper-${ZOOKEEPER_VERSION}.tar.gz
+PROMETHEUS_PACKAGE=prometheus-2.31.1.linux-amd64.tar.gz
+NODE_EXPORTER_PACKAGE=node_exporter-1.3.1.linux-amd64.tar.gz
+
+### Parameter for JDK 1.8
+JDK_PACKAGE=jdk-8u301-linux-x64.tar.gz
+JDK_DECOMPRESS_NAME=jdk1.8.0_301
+
+HOME_DIR=/home/ec2-user
+
+CURRENT_HOST=$(hostname -i)
+
+function init_env() {
+ HADOOP_DIR=${HOME_DIR}/hadoop
+ if [[ ! -d $HADOOP_DIR ]]; then
+ mkdir ${HADOOP_DIR}
+ fi
+
+ JAVA_HOME=/usr/local/java
+ JRE_HOME=${JAVA_HOME}/jre
+ ZOOKEEPER_HOME=${HADOOP_DIR}/zookeeper
+ OUT_LOG=${HOME_DIR}/shell.stdout
+
+ # extra prometheus env
+ PROMETHEUS_HOME=/home/ec2-user/prometheus
+ NODE_EXPORTER_HOME=/home/ec2-user/node_exporter
+
+ cat <<EOF >>~/.bash_profile
+## Set env variables
+### jdk env
+export JAVA_HOME=${JAVA_HOME}
+export JRE_HOME=${JRE_HOME}
+export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
+
+### zookeeper env
+export ZOOKEEPER_HOME=${ZOOKEEPER_HOME}
+
+### prometheus related env
+export PROMETHEUS_HOME=${PROMETHEUS_HOME}
+export NODE_EXPORTER_HOME=${NODE_EXPORTER_HOME}
+
+### export all path
+export PATH=${JAVA_HOME}/bin:${ZOOKEEPER_HOME}/bin:$PATH
+
+### other
+export HOME_DIR=${HOME_DIR}
+export OUT_LOG=$OUT_LOG
+EOF
+}
+
+if [[ ! -f ~/.inited_env ]]; then
+ logging info "Env variables not init, init it first ..."
+ init_env
+ touch ~/.inited_env
+else
+ logging warn "Env variables already inited, source it ..."
+fi
+source ~/.bash_profile
+exec 2>>${OUT_LOG}
+set -o pipefail
+# ================ Main Functions ======================
+function help() {
+ logging warn "Invalid input."
+ logging warn "Usage: ${BASH_SOURCE[0]}
+ --bucket-url /path/to/bucket/without/prefix
+ --region region-for-current-instance
+ --zk-num current-zookeeper-number"
+ exit 0
+}
+
+if [[ $# -ne 6 ]]; then
+ help
+fi
+
+while [[ $# != 0 ]]; do
+ if [[ $1 == "--bucket-url" ]]; then
+ # url same as: /xxx/kylin
+ BUCKET_SUFFIX=$2
+ elif [[ $1 == "--region" ]]; then
+ CURRENT_REGION=$2
+ elif [[ $1 == "--zk-num" ]]; then
+ ZK_NUM=$2
+ else
+ help
+ fi
+ shift
+ shift
+done
+
+PATH_TO_BUCKET=s3:/${BUCKET_SUFFIX}
+
+# Main Functions and Steps
+## prepare jdk env
+function prepare_jdk() {
+ logging info "Preparing Jdk ..."
+ if [[ -f ${HOME_DIR}/.prepared_jdk ]]; then
+ logging warn "Jdk already prepared, enjoy it."
+ return
+ fi
+
+ # copy jdk from s3 bucket to ec2 instances, so user need to upload jdk package first
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${JDK_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # unzip jdk: tar -C /extract/to/path -xzvf /path/to/archive.tar.gz
+ tar -zxf ${JDK_PACKAGE}
+ sudo mv ${JDK_DECOMPRESS_NAME} ${JAVA_HOME}
+ if [[ $? -ne 0 ]]; then
+ logging error "Java package was not installed well, pleas check ..."
+ exit 0
+ fi
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.prepared_jdk
+ logging info "Jdk is ready ..."
+}
+
+function init_jdk() {
+ if [[ -f ${HOME_DIR}/.inited_jdk ]]; then
+ logging warn "Jdk already inited, skip init ..."
+ return
+ fi
+ # this function is remove the unsupport tls rules in java which version of 1.8.291 and above
+ ## backup
+ cp -f $JAVA_HOME/jre/lib/security/java.security $JAVA_HOME/jre/lib/security/java.security.bak
+
+ ## modify the java.security file
+ sed -e "s/\ TLSv1,\ TLSv1.1,//g" -i $JAVA_HOME/jre/lib/security/java.security
+ logging info "Jdk inited ..."
+ touch ${HOME_DIR}/.inited_jdk
+}
+
+function prepare_zookeeper() {
+ logging info "Preparing zookeeper ..."
+ if [[ -f ${HOME_DIR}/.prepared_zookeeper ]]; then
+ logging warn "Zookeeper service already installed, restart it."
+ return
+ fi
+
+ if [[ -f ./${ZOOKEEPER_PACKAGE} ]]; then
+ logging warn "Zookeeper package already download, skip download it"
+ else
+ logging info "Downloading Zookeeper package ${ZOOKEEPER_PACKAGE} ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${ZOOKEEPER_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ # # wget cost lot time
+ # wget http://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/${ZOOKEEPER_PACKAGE}
+ fi
+
+ if [[ -d ${HOME_DIR}/zookeeper-${ZOOKEEPER_VERSION} ]]; then
+ logging warn "Zookeeper Package decompressed, skip decompress ..."
+ else
+ logging info "Decompress Zookeeper package ..."
+ tar -zxf ${ZOOKEEPER_PACKAGE}
+ fi
+
+ logging info "Zookeeper prepared ..."
+ touch ${HOME_DIR}/.prepared_zookeeper
+}
+
+function init_zookeeper() {
+ if [[ -f ${HOME_DIR}/.inited_zookeeper ]]; then
+ logging warn "Zookeeper already inited ..."
+ else
+ logging info "Init Zookeeper config ..."
+ # copy cfg to set fake zk cluster
+ cp zookeeper-${ZOOKEEPER_VERSION}/conf/zoo_sample.cfg zookeeper-${ZOOKEEPER_VERSION}/conf/zoo.cfg
+
+ cat <<EOF >zookeeper-${ZOOKEEPER_VERSION}/conf/zoo.cfg
+# zoo${ZK_NUM}.cfg
+tickTime=2000
+initLimit=10
+syncLimit=5
+dataDir=/tmp/zookeeper/zk${ZK_NUM}/data
+dataLogDir=/tmp/zookeeper/zk${ZK_NUM}/log
+clientPort=2181
+# server.${ZK_NUM}=${CURRENT_HOST}:2287:3387
+EOF
+ mkdir -p /tmp/zookeeper/zk${ZK_NUM}/log
+ mkdir -p /tmp/zookeeper/zk${ZK_NUM}/data
+ echo ${ZK_NUM} >>/tmp/zookeeper/zk${ZK_NUM}/data/myid
+
+ logging info "Moving ${HOME_DIR}/zookeeper-${ZOOKEEPER_VERSION} to ${ZOOKEEPER_HOME} ..."
+ mv ${HOME_DIR}/zookeeper-${ZOOKEEPER_VERSION} ${ZOOKEEPER_HOME}
+
+ logging warn "touch ${HOME_DIR}/.inited_zookeeper ..."
+ touch ${HOME_DIR}/.inited_zookeeper
+ fi
+ logging info "Zookeeper is ready ..."
+}
+
+function prepare_node_exporter() {
+ logging info "Preparing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.prepared_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already prepared, skip prepare ... "
+ return
+ fi
+
+ if [[ ! -f ${HOME_DIR}/${NODE_EXPORTER_PACKAGE} ]]; then
+ logging info "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} not downloaded, downloading it ..."
+ aws s3 cp ${PATH_TO_BUCKET}/tar/${NODE_EXPORTER_PACKAGE} ${HOME_DIR} --region ${CURRENT_REGION}
+ else
+ logging warn "NODE_EXPORTER package ${NODE_EXPORTER_PACKAGE} already download, skip download it."
+ fi
+ touch ${HOME_DIR}/.prepared_node_exporter
+ logging info "NODE_EXPORTER prepared ..."
+}
+
+function init_node_exporter() {
+ logging info "Initializing node_exporter ..."
+ if [[ -f ${HOME_DIR}/.inited_node_exporter ]]; then
+ logging warn "NODE_EXPORTER already inited, skip init ... "
+ return
+ fi
+
+ if [[ ! -f ${NODE_EXPORTER_HOME} ]]; then
+ logging info "NODE_EXPORTER home ${NODE_EXPORTER_HOME} not ready, decompressing ${NODE_EXPORTER_PACKAGE} ..."
+ tar -zxf ${HOME_DIR}/${NODE_EXPORTER_PACKAGE}
+ mv ${HOME_DIR}/${NODE_EXPORTER_PACKAGE%.tar.gz} ${NODE_EXPORTER_HOME}
+ else
+ logging warn "NODE_EXPORTER home ${PROMETHEUS_PACKAGE} already ready."
+ fi
+ touch ${HOME_DIR}/.inited_node_exporter
+ logging info "NODE_EXPORTER inited ..."
+}
+
+function start_node_exporter() {
+ # NOTE: default node_exporter port 9100
+ logging info "Start node_exporter ..."
+ nohup ${NODE_EXPORTER_HOME}/node_exporter >> ${NODE_EXPORTER_HOME}/node.log 2>&1 &
+}
+
+function prepare_packages() {
+ if [[ -f ${HOME_DIR}/.prepared_packages ]]; then
+ logging warn "Packages already prepared, skip prepare ..."
+ return
+ fi
+
+ prepare_jdk
+ init_jdk
+
+ # add extra monitor service
+ prepare_node_exporter
+ init_node_exporter
+
+ prepare_zookeeper
+ init_zookeeper
+
+ touch ${HOME_DIR}/.prepared_packages
+ logging info "All need packages are ready ..."
+}
+
+function start_services_on_other() {
+ # Note: start zookeeper after created all needed nodes, because need to update every zk cfg.
+
+ # start extra monitor service
+ # NOTE: prometheus server will start after all node_exporter on every node started.
+ start_node_exporter
+
+}
+
+function main() {
+ prepare_packages
+ start_services_on_other
+}
+
+main
diff --git a/bin/init.sh b/bin/init.sh
new file mode 100755
index 0000000..88fb704
--- /dev/null
+++ b/bin/init.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+function info() {
+ echo -e "\033[32m$@\033[0m"
+}
+
+function warn() {
+ echo -e "\033[33m$@\033[0m"
+}
+
+function error() {
+ echo -e "\033[31m$@\033[0m"
+}
+
+function logging() {
+ case $1 in
+ "info") shift; info $@ ;;
+ "warn") shift; warn $@ ;;
+ "error") shift; error $@ ;;
+ *) echo -e $@ ;;
+ esac
+}
+
+if [[ -z "$KYLIN_TPCH_HOME" ]]; then
+ dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+ export KYLIN_TPCH_HOME=`cd "${dir}/../"; pwd`
+ logging info "KYLIN_TPCH_HOME not set, will use $KYLIN_TPCH_HOME as KYLIN_TPCH_HOME."
+fi
+
+function check_python_version {
+ pyv=$(python3 --version 2>&1)
+ currentver=${pyv:7}
+ logging info "Current Python version is $currentver."
+
+ requiredver=3.6.6
+
+ if [[ "$(printf '%s\n' "$requiredver" "$currentver" | sort -V | head -n1)" != "$requiredver" ]]; then
+ logging error "${currentver} less than required Python version ${requiredver}."
+ exit 0
+ fi
+}
+
+function install_env {
+ check_python_version
+ if [[ ! -d $KYLIN_TPCH_HOME/venv ]]; then
+ python3 -m venv $KYLIN_TPCH_HOME/venv
+ source $KYLIN_TPCH_HOME/venv/bin/activate
+ logging info "Install dependencies ..."
+ pip3 install -r $KYLIN_TPCH_HOME/requirements.txt
+ else
+ logging warn "$KYLIN_TPCH_HOME/.venv already existing, skip install again."
+ fi
+ logging info "Please use 'source venv/bin/activate' to activate venv and execute commands."
+ logging info "Please use 'python ./deploy.py --h[|--help]' to check the usage."
+ logging info "Enjoy it and have fun."
+}
+
+function main {
+ install_env
+}
+
+main
diff --git a/client/__init__.py b/client/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/client/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/client/basic.py b/client/basic.py
new file mode 100644
index 0000000..84bfdcb
--- /dev/null
+++ b/client/basic.py
@@ -0,0 +1,133 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import time
+
+import requests
+from retrying import retry
+
+
+logger = logging.getLogger(__name__)
+
+
+def _result(exception):
+ return isinstance(exception, requests.exceptions.ConnectTimeout)
+
+
+class BasicHttpClient:
+ _host = None
+ _port = None
+
+ _headers = {}
+
+ _auth = ('ADMIN', 'KYLIN')
+
+ _inner_session = requests.Session()
+
+ def __init__(self, host, port):
+ if not host or not port:
+ raise ValueError('init http client failed')
+
+ self._host = host
+ self._port = port
+
+ def token(self, token):
+ self._headers['Authorization'] = 'Basic {token}'.format(token=token)
+
+ def auth(self, username, password):
+ self._auth = (username, password)
+
+ def header(self, name, value):
+ self._headers[name] = value
+
+ def headers(self, headers):
+ self._headers = headers
+
+ def _request(self, method, url, params=None, data=None, json=None, # pylint: disable=too-many-arguments
+ files=None, headers=None, stream=False, to_json=True, inner_session=False, timeout=60, content=False,
+ raw_response=False):
+ if inner_session:
+ return self._request_with_session(self._inner_session, method, url,
+ params=params,
+ data=data,
+ json=json,
+ files=files,
+ headers=headers,
+ stream=stream,
+ to_json=to_json,
+ timeout=timeout,
+ content=content,
+ raw_response=raw_response
+ )
+ with requests.Session() as session:
+ session.auth = self._auth
+ return self._request_with_session(session, method, url,
+ params=params,
+ data=data,
+ json=json,
+ files=files,
+ headers=headers,
+ stream=stream,
+ to_json=to_json,
+ timeout=timeout,
+ content=content,
+ raw_response=raw_response
+ )
+
+ @retry(stop_max_attempt_number=3, wait_random_min=100, wait_random_max=200, retry_on_exception=_result)
+ def _request_with_session(self, session, method, url, params=None, data=None, # pylint: disable=too-many-arguments
+ json=None, files=None, headers=None, stream=False,
+ to_json=True, timeout=60, content=False, raw_response=False):
+ if headers is None:
+ headers = self._headers
+ resp = session.request(method, url,
+ params=params,
+ data=data,
+ json=json,
+ files=files,
+ headers=headers,
+ stream=stream,
+ timeout=timeout,
+ verify=False
+ )
+
+ try:
+ if raw_response:
+ return resp
+ if stream:
+ return resp.raw
+ if not resp.content:
+ return None
+ if content:
+ return resp.content
+ if to_json:
+ data = resp.json()
+ resp.raise_for_status()
+ return data.get('data', data)
+ return resp.text
+ except requests.exceptions.ReadTimeout as timeout_error:
+ logger.error(timeout_error)
+ time.sleep(60)
+ except requests.HTTPError as http_error:
+ err_msg = f"{str(http_error)} [return code: {data.get('code', '')}]-[{data.get('msg', '')}]\n" \
+ f"{data.get('stacktrace', '')}"
+ logger.error(err_msg)
+ raise requests.HTTPError(err_msg, request=http_error.request, response=http_error.response, )
+ except Exception as error:
+ logger.error(str(error))
+ raise error
diff --git a/client/lightning.py b/client/lightning.py
new file mode 100644
index 0000000..52d7181
--- /dev/null
+++ b/client/lightning.py
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import time
+
+from client.basic import BasicHttpClient
+
+logger = logging.getLogger(__name__)
+
+
+class LightningHttpClient(BasicHttpClient):
+ _base_url = 'http://{host}:{port}'
+
+ def __init__(self, host, port):
+ super().__init__(host, port)
+ self._headers = {
+ 'Accept': 'application/json',
+ 'Accept-Language': 'en-US',
+ 'Content-Type': 'application/json;charset=utf-8'
+ }
+ self._base_url = self._base_url.format(host=self._host, port=self._port)
+
+ def set_headers(self, key, value=None):
+ if value is not None:
+ self._headers[key] = value
+ else:
+ if key in self._headers.keys():
+ self._headers.pop(key)
+
+ def set_base_url(self, url):
+ self._base_url = url
+
+ def await_kylin_start(self, check_action=None, timeout=600, check_times=30, **kwargs):
+ start_time = time.time()
+ already_check_times = 0
+ while time.time() - start_time < timeout:
+ try:
+ while already_check_times < check_times:
+ res = check_action(**kwargs)
+ assert res
+ already_check_times = already_check_times + 1
+ logger.info(f'Already check {already_check_times} times')
+ time.sleep(1)
+ return True
+ except Exception as e:
+ logger.debug('Kylin can not access now {}, wait 10s'.format(e))
+ already_check_times = 0
+ time.sleep(10)
+ logger.debug('Kylin can not access after {}s'.format(timeout))
+ return False
+
+ def login(self, username, password, user_session=False):
+ """
+ use username and password login
+ :param username: string, target group name
+ :param password: array, the users add to group
+ :param user_session: boolean, true for using login session to execute
+ :return:
+ """
+ url = '/kylin/api/user/authentication'
+ self.auth(username, password)
+ resp = self._request('POST', url, inner_session=user_session)
+ return resp
+
+ def get_session(self):
+ return self._inner_session
+
+ def check_login_state(self):
+ url = '/kylin/api/user/authentication'
+ resp = self._request('GET', url, inner_session=False)
+ return resp
+
+ def execute_query(self, project, sql, offset=0, limit=500, accept_partial=False, user_session=False, timeout=360):
+ """
+ :param project: string, the name of project
+ :param sql: query sql
+ :param offset: offset of returned result
+ :param limit: limit of returned result
+ :param timeout: session timeout time
+ :return:
+ """
+ url = '/kylin/api/query'
+ payload = {
+ 'limit': limit,
+ 'offset': offset,
+ 'project': project,
+ 'acceptPartial': accept_partial,
+ 'sql': sql,
+ }
+ resp = self._request('POST', url, json=payload, inner_session=user_session, timeout=timeout)
+ return resp
+
+ def _request(self, method, url, **kwargs): # pylint: disable=arguments-differ
+ return super()._request(method, self._base_url + url, **kwargs)
+
+
+def connect(**conf):
+ _host = conf.get('host')
+ _port = conf.get('port')
+
+ return LightningHttpClient(_host, _port)
diff --git a/cloudformation_templates/ec2-cluster-kylin4-template.yaml b/cloudformation_templates/ec2-cluster-kylin4-template.yaml
new file mode 100644
index 0000000..5b77e12
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-kylin4-template.yaml
@@ -0,0 +1,252 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Kylin4 nodes for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ DbHost:
+ Type: String
+ Description: Must be created at pre-step, db from ec2 instance
+ DbUser:
+ Type: String
+ Description: Db user for kylin4 to access RDS
+ Default: root
+ DbPassword:
+ Type: String
+ Description: Db password for kylin4 to access RDS
+ DbPort:
+ Type: String
+ Description: Db port for kylin4 to access RDS
+ Default: 3306
+ ZookeepersHost:
+ Type: String
+ Description: Must be created at pre-step
+ SparkMasterNodeHost:
+ Type: String
+ Description: Spark Master host for kylin 4
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ AssociatedPublicIp:
+ Type: String
+ Default: true
+ AllowedValues:
+ - true
+ - false
+ EMREc2KeyName:
+ Type: String
+
+ ClusterNum:
+ Type: String
+ Default: default
+
+ IsScaled:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ Description: set the flag for Mode
+
+# Optional Parameters
+ LocalCacheSoftAffinity:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+ BucketFullPath:
+ Type: String
+ BucketPath:
+ Type: String
+ Description: Url without prefix s3:/
+ Kylin4ScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-kylin4.sh
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ InstanceTypeForTest:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.2xlarge
+ Ec2VolumnTypeForKylin4Node:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForKylin4Node:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 100
+
+ Ec2VolumnTypeForKylin4NodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+
+ Ec2VolumeSizeForKylin4NodeForTest:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 30
+
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ NotNullDbHost: !Not [!Equals [!Ref DbHost, ""]]
+ NotNullDbPass: !Not [!Equals [!Ref DbPassword, ""]]
+ NotNullDbUser: !Not [!Equals [!Ref DbUser, ""]]
+ NotNullZkHost: !Not [!Equals [!Ref ZookeepersHost, ""]]
+ NotNullSparkMasterHost: !Not [!Equals [!Ref SparkMasterNodeHost, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ - !Condition NotNullDbHost
+ - !Condition NotNullDbPass
+ - !Condition NotNullDbUser
+ - !Condition NotNullZkHost
+ - !Condition NotNullSparkMasterHost
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfKylin4:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Kylin4 Node for kylin4
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref InstanceType
+ - !Ref InstanceTypeForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Kylin4 node
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForKylin4Node
+ VolumeType: !Ref Ec2VolumnTypeForKylin4Node
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForKylin4NodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForKylin4NodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateKylin4ScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateKylin4ScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --db-host ${PrivateDbHost} --db-password ${PrivateDbPass} --db-user ${PrivateDbUser} --db-port ${PrivateDbPort} --local-soft ${PrivateLocalCacheSoftAffinity} --cluster-num ${PrivateClusterNum} --is-scaled ${PrivateIsScaled}
+ echo " Kylin4 is ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateKylin4ScriptFileName: !Ref Kylin4ScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateDbHost: !Ref DbHost
+ PrivateDbPass: !Ref DbPassword
+ PrivateDbUser: !Ref DbUser
+ PrivateDbPort: !Ref DbPort
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+ PrivateClusterNum: !Ref ClusterNum
+ PrivateIsScaled: !Ref IsScaled
+
+Outputs:
+ IdOfInstance:
+ Description: the id of Kylin4 Node Innstance
+ Value: !Ref Ec2InstanceOfKylin4
+ Kylin4Ec2InstancePrivateIp:
+ Description: Kylin4 Instance Private IP
+ Value: !GetAtt Ec2InstanceOfKylin4.PrivateIp
+ Kylin4Ec2InstancePublicIp:
+ Description: Kylin4 Instance Public IP to acess
+ Value: !GetAtt Ec2InstanceOfKylin4.PublicIp
+ Condition: IsAssociatedPublicIp
+ Kylin4SubnetIdDependsOnDNode:
+ Description: SubnetId in the vpc for Slave Node
+ Value: !Ref SubnetId
+ Kylin4SecurityGroupIdDependsOnDNode:
+ Value: !Ref SecurityGroupId
+ Kylin4SparkMasterHost:
+ Description: Spark Master host for Kylin 4
+ Value: !Ref SparkMasterNodeHost
+ Kylin4ZookeeperHosts:
+ Description: Zookeeper hosts for Kylin 4
+ Value: !Ref ZookeepersHost
diff --git a/cloudformation_templates/ec2-cluster-kylin4.yaml b/cloudformation_templates/ec2-cluster-kylin4.yaml
new file mode 100644
index 0000000..dcfd775
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-kylin4.yaml
@@ -0,0 +1,251 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Kylin4 nodes for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ DbHost:
+ Type: String
+ Description: Must be created at pre-step, db from ec2 instance
+ DbUser:
+ Type: String
+ Description: Db user for kylin4 to access RDS
+ Default: root
+ DbPassword:
+ Type: String
+ Description: Db password for kylin4 to access RDS
+ DbPort:
+ Type: String
+ Description: Db port for kylin4 to access RDS
+ Default: 3306
+ ZookeepersHost:
+ Type: String
+ Description: Must be created at pre-step
+ SparkMasterNodeHost:
+ Type: String
+ Description: Spark Master host for kylin 4
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ AssociatedPublicIp:
+ Type: String
+ Default: true
+ AllowedValues:
+ - true
+ - false
+ EMREc2KeyName:
+ Type: String
+
+ ClusterNum:
+ Type: String
+ Default: default
+
+ IsScaled:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ Description: set the flag for Mode, if not test then use product's configuration
+
+# Optional Parameters
+ LocalCacheSoftAffinity:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+ BucketFullPath:
+ Type: String
+ BucketPath:
+ Type: String
+ Description: Url without prefix s3:/
+ Kylin4ScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-kylin4.sh
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ InstanceTypeForTest:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.2xlarge
+ Ec2VolumnTypeForKylin4Node:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForKylin4Node:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 100
+
+ Ec2VolumnTypeForKylin4NodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+
+ Ec2VolumeSizeForKylin4NodeForTest:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 30
+
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ NotNullDbHost: !Not [!Equals [!Ref DbHost, ""]]
+ NotNullDbPass: !Not [!Equals [!Ref DbPassword, ""]]
+ NotNullDbUser: !Not [!Equals [!Ref DbUser, ""]]
+ NotNullZkHost: !Not [!Equals [!Ref ZookeepersHost, ""]]
+ NotNullSparkMasterHost: !Not [!Equals [!Ref SparkMasterNodeHost, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ - !Condition NotNullDbHost
+ - !Condition NotNullDbPass
+ - !Condition NotNullDbUser
+ - !Condition NotNullZkHost
+ - !Condition NotNullSparkMasterHost
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfKylin4:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Kylin4 Node for kylin4
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref InstanceType
+ - !Ref InstanceTypeForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Kylin4 node
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForKylin4Node
+ VolumeType: !Ref Ec2VolumnTypeForKylin4Node
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForKylin4NodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForKylin4NodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateKylin4ScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateKylin4ScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --db-host ${PrivateDbHost} --db-password ${PrivateDbPass} --db-user ${PrivateDbUser} --db-port ${PrivateDbPort} --local-soft ${PrivateLocalCacheSoftAffinity} --cluster-num ${PrivateClusterNum} --is-scaled ${PrivateIsScaled}
+ echo " Kylin4 is ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateKylin4ScriptFileName: !Ref Kylin4ScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateDbHost: !Ref DbHost
+ PrivateDbPass: !Ref DbPassword
+ PrivateDbUser: !Ref DbUser
+ PrivateDbPort: !Ref DbPort
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+ PrivateClusterNum: !Ref ClusterNum
+ PrivateIsScaled: !Ref IsScaled
+
+Outputs:
+ IdOfInstance:
+ Description: the id of Kylin4 Node Innstance
+ Value: !Ref Ec2InstanceOfKylin4
+ Kylin4Ec2InstancePrivateIp:
+ Description: Kylin4 Instance Private IP
+ Value: !GetAtt Ec2InstanceOfKylin4.PrivateIp
+ Kylin4Ec2InstancePublicIp:
+ Description: Kylin4 Instance Public IP to acess
+ Value: !GetAtt Ec2InstanceOfKylin4.PublicIp
+ Condition: IsAssociatedPublicIp
+ Kylin4SubnetIdDependsOnDNode:
+ Description: SubnetId in the vpc for Slave Node
+ Value: !Ref SubnetId
+ Kylin4SecurityGroupIdDependsOnDNode:
+ Value: !Ref SecurityGroupId
+ Kylin4SparkMasterHost:
+ Description: Spark Master host for Kylin 4
+ Value: !Ref SparkMasterNodeHost
+ Kylin4ZookeeperHosts:
+ Description: Zookeeper hosts for Kylin 4
+ Value: !Ref ZookeepersHost
diff --git a/cloudformation_templates/ec2-cluster-rds.yaml b/cloudformation_templates/ec2-cluster-rds.yaml
new file mode 100644
index 0000000..24ea418
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-rds.yaml
@@ -0,0 +1,149 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Kylin4 nodes for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ Ec2DbIdentifier:
+ Default: ec2-rds-kylin4
+ Type: String
+
+ Ec2DbUser:
+ NoEcho: 'true'
+ Description: Username for MySQL database access
+ Type: String
+ MinLength: '1'
+ MaxLength: '16'
+ AllowedPattern: '[a-zA-Z][a-zA-Z0-9]*'
+ Default: root
+ Ec2DbPassword:
+ NoEcho: 'true'
+ Type: String
+ MinLength: '8'
+ MaxLength: '41'
+ AllowedPattern: '[a-zA-Z0-9]*'
+ ConstraintDescription: must contain only alphanumeric characters.
+ Description: Must be created at pre-step and this can be modified in the script not yaml file.
+ Ec2DbPort:
+ Type: String
+ Default: 3306
+ AllowedValues:
+ - 3306
+
+ SecurityGroupId:
+ Type: String
+ SubnetGroupName:
+ Type: String
+
+ RDSEngine:
+ Type: String
+ Default: mysql
+ AllowedValues:
+ - mysql
+ RDSEngineVersion:
+ Type: String
+ Default: 5.7.25
+
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ Description: set the flag for Mode, if not `test` then use `product`
+
+# Optional Parameters
+ EnableIAMAuthentication:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+ Zone:
+ Type: String
+ Default: cn-northwest-1b
+ AllowedValues:
+ - cn-northwest-1a
+ - cn-northwest-1b
+ DbName:
+ Type: String
+ Description: Initilize a db in rds
+ Default: kylin
+ AllowedValues:
+ - kylin
+ TypeOfRDS:
+ Description: RDS instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: db.t3.micro
+ AllowedValues:
+ - db.t3.micro
+ - db.m5.xlarge
+ VolumnTypeForRDS:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ VolumeSizeForRDS:
+ Type: Number
+ Default: 10
+ MinValue: 5
+ MaxValue: 20
+
+Conditions:
+ NotNullSecurityGroupID: !Not [!Equals [!Ref SecurityGroupId, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+
+Resources:
+ Ec2RDSInstance:
+ Type: AWS::RDS::DBInstance
+ DeletionPolicy: Delete
+ Condition: NotNullSecurityGroupID
+ Properties:
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: RDS for kylin4
+ AllocatedStorage: !Ref VolumeSizeForRDS
+ AvailabilityZone: !Ref Zone
+ DBSubnetGroupName: !Ref SubnetGroupName
+ # The number of days for which automated backups are retained. Setting this parameter to a positive number enables backups.
+ # Setting this parameter to 0 disables automated backups.
+ BackupRetentionPeriod: 0
+ DBInstanceClass: !Ref TypeOfRDS
+ DBInstanceIdentifier: !Ref Ec2DbIdentifier
+ DBName: !Ref DbName
+ VPCSecurityGroups:
+ - !Ref SecurityGroupId
+ EnableIAMDatabaseAuthentication: !Ref EnableIAMAuthentication
+ Engine: !Ref RDSEngine
+ EngineVersion: !Ref RDSEngineVersion
+ MasterUsername: !Ref Ec2DbUser
+ MasterUserPassword: !Ref Ec2DbPassword
+ Port: !Ref Ec2DbPort
+ StorageType: !Ref VolumnTypeForRDS
+
+Outputs:
+ DbHost:
+ Description: Db Endpoints Host
+ Value: !GetAtt Ec2RDSInstance.Endpoint.Address
+ DbPort:
+ Description: Db Endpoints Port
+ Value: !GetAtt Ec2RDSInstance.Endpoint.Port
+
diff --git a/cloudformation_templates/ec2-cluster-spark-master.yaml b/cloudformation_templates/ec2-cluster-spark-master.yaml
new file mode 100644
index 0000000..dfd5a8b
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-spark-master.yaml
@@ -0,0 +1,219 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create SparkMaster nodes for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ AssociatedPublicIp:
+ Type: String
+ Default: true
+ AllowedValues:
+ - true
+ - false
+ EMREc2KeyName:
+ Type: String
+ DbHost:
+ Type: String
+ Description: the host of RDS instance
+ DbUser:
+ Type: String
+ Description: the user to access RDS instance
+ DbPassword:
+ Type: String
+ Description: the password to access RDS instance
+ DbPort:
+ Type: String
+ Description: the port to access RDS instance
+ Default: 3306
+
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ Description: set the flag for Mode, if not test use product's configuration
+
+# Optional Parameters
+ LocalCacheSoftAffinity:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+ BucketFullPath:
+ Type: String
+ BucketPath:
+ Type: String
+ Description: Url without prefix s3:/
+ SparkMasterScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-spark-master.sh
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ InstanceTypeForTest:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.2xlarge
+ Ec2VolumnTypeForSparkMasterNode:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForSparkMasterNode:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 100
+
+ Ec2VolumnTypeForSparkMasterNodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+
+ Ec2VolumeSizeForSparkMasterNodeForTest:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 30
+
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfSparkMaster:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Spark Master Node for kylin4
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref InstanceType
+ - !Ref InstanceTypeForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for SparkMaster node in benchmark
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSparkMasterNode
+ VolumeType: !Ref Ec2VolumnTypeForSparkMasterNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSparkMasterNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForSparkMasterNodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateSparkMasterScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateSparkMasterScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --local-soft ${PrivateLocalCacheSoftAffinity} --db-port ${PrivateDbPort} --db-host ${PrivateDbHost} --db-user ${PrivateDbUser} --db-password ${PrivateDbPassword}
+ echo " Spark Master is ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateSparkMasterScriptFileName: !Ref SparkMasterScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+ PrivateDbPort: !Ref DbPort
+ PrivateDbHost: !Ref DbHost
+ PrivateDbUser: !Ref DbUser
+ PrivateDbPassword: !Ref DbPassword
+
+Outputs:
+ IdOfInstance:
+ Description: the id of SparkMaster Node Instance
+ Value: !Ref Ec2InstanceOfSparkMaster
+ SparkMasterNodeHost:
+ Description: SparkMaster host is same as private IP
+ Value: !GetAtt Ec2InstanceOfSparkMaster.PrivateIp
+ SparkMasterEc2InstancePrivateIp:
+ Description: SparkMaster Instance Private IP
+ Value: !GetAtt Ec2InstanceOfSparkMaster.PrivateIp
+ SparkMasterEc2InstancePublicIp:
+ Description: SparkMaster Instance Public IP to acess
+ Value: !GetAtt Ec2InstanceOfSparkMaster.PublicIp
+ Condition: IsAssociatedPublicIp
+ SparkMasterEc2InstanceProfileId:
+ Value: !Ref InstanceProfileId
+ SparkMasterSubnetIdDependsOnDNode:
+ Value: !Ref SubnetId
+ SparkMasterSecurityGroupIdDependsOnDNode:
+ Value: !Ref SecurityGroupId
diff --git a/cloudformation_templates/ec2-cluster-spark-slave-template.yaml b/cloudformation_templates/ec2-cluster-spark-slave-template.yaml
new file mode 100644
index 0000000..0e6260c
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-spark-slave-template.yaml
@@ -0,0 +1,210 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Other needed worker nodes for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ SparkMasterNodeHost:
+ Type: String
+ Description: Must be created at pre-step
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ EMREc2KeyName:
+ Type: String
+
+ AssociatedPublicIp:
+ Type: String
+ Description: whether to associated public ip for instances
+ Default: false
+ AllowedValues:
+ - true
+ - false
+
+ WorkerNum:
+ Type: String
+ Description: mark woker number for spark worker
+
+ # Optional Parameters
+ LocalCacheSoftAffinity:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ WaitingTime:
+ Type: Number
+ Default: 600
+ Description: Waiting time for start slave worker
+ BucketFullPath:
+ Type: String
+ BucketPath:
+ Type: String
+ Description: Url without prefix s3:/
+ ScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-spark-slave.sh
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: Must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ Ec2VolumnTypeForSlaveNode:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForSlaveNode:
+ Type: Number
+ Default: 400
+ MinValue: 30
+ MaxValue: 400
+ Ec2VolumnTypeForSlaveNodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ Ec2VolumeSizeForSlaveNodeForTest:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 30
+
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ NotNullSparkMasterNodeHost: !Not [!Equals [!Ref SparkMasterNodeHost, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ - !Condition NotNullSparkMasterNodeHost
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfSlave:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Description: Scale node WorkerNum
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Slave Scale
+ - !Ref WorkerNum
+ - Node for kylin4
+ InstanceType: !Ref InstanceType
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Scale Slave Node NetworkInterfaces
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNodeForTest
+ DeleteOnTermination: true
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvdb
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - !Ref AWS::NoValue
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateSlaveScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateSlaveScriptFileName} --bucket-url ${PrivateBucketPath} --master-host ${PrivateMasterHost} --worker-number ${WorkerNum} --region ${PrivateRegion} --waiting-time ${PrivateWaitingTime} --mode ${WorkerMode} --local-soft ${PrivateLocalCacheSoftAffinity}
+ - PrivateMasterHost: !Ref SparkMasterNodeHost
+ WorkerNum: !Ref WorkerNum
+ PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateSlaveScriptFileName: !Ref ScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateWaitingTime: !Ref WaitingTime
+ WorkerMode: !Ref Ec2Mode
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+
+Outputs:
+ IdOfInstance:
+ Description: the id of Slave Instance
+ Value: !Ref Ec2InstanceOfSlave
+ SlaveEc2InstancePrivateIp:
+ Description: the Slave Instance Private IP
+ Value: !GetAtt Ec2InstanceOfSlave.PrivateIp
+ SlaveEc2InstancePublicIp:
+ Description: the Slave Instance Public IP
+ Value: !GetAtt Ec2InstanceOfSlave.PublicIp
+ Condition: IsAssociatedPublicIp
diff --git a/cloudformation_templates/ec2-cluster-spark-slave.yaml b/cloudformation_templates/ec2-cluster-spark-slave.yaml
new file mode 100644
index 0000000..ba5a923
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-spark-slave.yaml
@@ -0,0 +1,377 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Slave nodes for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ SparkMasterNodeHost:
+ Type: String
+ Description: Must be created at pre-step
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ EMREc2KeyName:
+ Type: String
+
+ AssociatedPublicIp:
+ Type: String
+ Description: whether to associated public ip for instances
+ Default: false
+ AllowedValues:
+ - true
+ - false
+
+# Optional Parameters
+ LocalCacheSoftAffinity:
+ Type: String
+ Default: false
+ AllowedValues:
+ - true
+ - false
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+
+ WaitingTime:
+ Type: Number
+ Default: 50
+ Description: Waiting time for start slave worker
+ BucketFullPath:
+ Type: String
+ Description: example s3://xxx/kylin
+ BucketPath:
+ Type: String
+ Description: example /xxx/kylin which Url is without prefix s3:/
+ SlaveScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-spark-slave.sh
+ InstanceType:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.xlarge
+ - m5.2xlarge
+ - m5.4xlarge
+ InstanceTypeForTest:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.2xlarge
+ Ec2VolumnTypeForSlaveNode:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForSlaveNode:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 400
+
+ Ec2VolumnTypeForSlaveNodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ Ec2VolumeSizeForSlaveNodeForTest:
+ Type: Number
+ Default: 30
+ MinValue: 30
+ MaxValue: 30
+
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ NotNullSparkMasterNodeHost: !Not [!Equals [!Ref SparkMasterNodeHost, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ - !Condition NotNullSparkMasterNodeHost
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfSlave01:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Description: default slave01 node is created for query or job
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Slave01 Node for kylin4
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref InstanceType
+ - !Ref InstanceTypeForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Slave01 node in benchmark
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNodeForTest
+ DeleteOnTermination: true
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvdb
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - !Ref AWS::NoValue
+
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateSlaveScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateSlaveScriptFileName} --bucket-url ${PrivateBucketPath} --master-host ${PrivateMasterHost} --worker-number ${WorkerNum} --region ${PrivateRegion} --waiting-time ${PrivateWaitingTime} --mode ${WorkerMode} --local-soft ${PrivateLocalCacheSoftAffinity}
+ - PrivateMasterHost: !Ref SparkMasterNodeHost
+ WorkerNum: 1
+ PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateSlaveScriptFileName: !Ref SlaveScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateWaitingTime: !Ref WaitingTime
+ WorkerMode: !Ref Ec2Mode
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+ Ec2InstanceOfSlave02:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Slave02 Node for Kylin4
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref InstanceType
+ - !Ref InstanceTypeForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Slave02 node in benchmark
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNodeForTest
+ DeleteOnTermination: true
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvdb
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - !Ref AWS::NoValue
+
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateSlaveScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateSlaveScriptFileName} --bucket-url ${PrivateBucketPath} --master-host ${PrivateMasterHost} --worker-number ${WorkerNum} --region ${PrivateRegion} --waiting-time ${PrivateWaitingTime} --mode ${WorkerMode} --local-soft ${PrivateLocalCacheSoftAffinity}
+ - PrivateMasterHost: !Ref SparkMasterNodeHost
+ WorkerNum: 2
+ PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateSlaveScriptFileName: !Ref SlaveScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateWaitingTime: !Ref WaitingTime
+ WorkerMode: !Ref Ec2Mode
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+
+ Ec2InstanceOfSlave03:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Slave03 Node for Kylin4
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref InstanceType
+ - !Ref InstanceTypeForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Slave03 node in benchmark
+ DeleteOnTermination: true
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNodeForTest
+ DeleteOnTermination: true
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvdb
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForSlaveNode
+ VolumeType: !Ref Ec2VolumnTypeForSlaveNode
+ DeleteOnTermination: true
+ - !Ref AWS::NoValue
+
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateSlaveScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateSlaveScriptFileName} --bucket-url ${PrivateBucketPath} --master-host ${PrivateMasterHost} --worker-number ${WorkerNum} --region ${PrivateRegion} --waiting-time ${PrivateWaitingTime} --mode ${WorkerMode} --local-soft ${PrivateLocalCacheSoftAffinity}
+ - PrivateMasterHost: !Ref SparkMasterNodeHost
+ WorkerNum: 3
+ PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateSlaveScriptFileName: !Ref SlaveScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateWaitingTime: !Ref WaitingTime
+ WorkerMode: !Ref Ec2Mode
+ PrivateLocalCacheSoftAffinity: !Ref LocalCacheSoftAffinity
+
+
+Outputs:
+ IdOfInstanceSlave01:
+ Description: the id of Slave 01 Node Instance if created
+ Value: !Ref Ec2InstanceOfSlave01
+ Slave01Ec2InstancePrivateIp:
+ Description: Slave01 Instance Private IP
+ Value: !GetAtt Ec2InstanceOfSlave01.PrivateIp
+ Slave01Ec2InstancePublicIp:
+ Description: Slave01 Instance Public IP
+ Value: !GetAtt Ec2InstanceOfSlave01.PublicIp
+ Condition: IsAssociatedPublicIp
+
+ IdOfInstanceSlave02:
+ Description: the id of Slave 02 Node Instance if created
+ Value: !Ref Ec2InstanceOfSlave02
+ Slave02Ec2InstancePrivateIp:
+ Description: Slave02 Instance Private IP
+ Value: !GetAtt Ec2InstanceOfSlave02.PrivateIp
+ Slave02Ec2InstancePublicIp:
+ Description: Slave02 Instance Public IP
+ Value: !GetAtt Ec2InstanceOfSlave02.PublicIp
+ Condition: IsAssociatedPublicIp
+
+ IdOfInstanceSlave03:
+ Description: the id of Slave 03 Node Instance if created
+ Value: !Ref Ec2InstanceOfSlave03
+ Slave03Ec2InstancePrivateIp:
+ Description: Slave03 Instance Private IP
+ Value: !GetAtt Ec2InstanceOfSlave03.PrivateIp
+ Slave03Ec2InstancePublicIp:
+ Description: Slave03 Instance Public IP
+ Value: !GetAtt Ec2InstanceOfSlave03.PublicIp
+ Condition: IsAssociatedPublicIp
diff --git a/cloudformation_templates/ec2-cluster-static-services.yaml b/cloudformation_templates/ec2-cluster-static-services.yaml
new file mode 100644
index 0000000..6916c63
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-static-services.yaml
@@ -0,0 +1,211 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Cluster for ec2 instances and deploy related services, prometheus/hivemetastore/jdk/node exporter
+Parameters:
+ # Must passed Parameters
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+ DbHost:
+ Type: String
+ Description: Db Host for RDS instance
+
+ DbUser:
+ Type: String
+ Description: Db User for RDS instance
+ DbPassword:
+ Type: String
+ Description: Db password for RDS instance
+ DbPort:
+ Type: String
+ Description: Db port for RDS instance
+
+ EMREc2KeyName:
+ Type: String
+
+# Optional Parameters
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ AssociatedPublicIp:
+ Type: String
+ Default: false
+ AllowedValues:
+ - false
+ - true
+ BucketFullPath:
+ Type: String
+ Description: example s3://xxx/kylin
+ BucketPath:
+ Type: String
+ Description: example /xxx/kylin which Url without prefix s3:/
+ StaticServicesScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-static-services.sh
+ Ec2InstanceTypeForStaticServices:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.2xlarge
+ AllowedValues:
+ - m5.2xlarge
+ - m5.4xlarge
+ Ec2InstanceTypeForStaticServicesForTest:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.xlarge
+ AllowedValues:
+ - m5.xlarge
+ Ec2VolumeSizeForStaticServicesNode:
+ Type: Number
+ Default: 20
+ MinValue: 20
+ MaxValue: 30
+ Ec2VolumnTypeForStaticServicesNode:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForStaticServicesNodeForTest:
+ Type: Number
+ Default: 20
+ MinValue: 20
+ MaxValue: 20
+ Ec2VolumnTypeForStaticServicesNodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfStaticServicesNode:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: StaticServices Node
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref Ec2InstanceTypeForStaticServices
+ - !Ref Ec2InstanceTypeForStaticServicesForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Master node in benchmark
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ DeleteOnTermination: true
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForStaticServicesNode
+ VolumeType: !Ref Ec2VolumnTypeForStaticServicesNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForStaticServicesNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForStaticServicesNodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateStaticServicesScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateStaticServicesScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --db-host ${PrivateDbHost} --db-password ${PrivateDbPassword} --db-user ${PrivateDbUser} --db-port ${PrivateDbPort}
+ echo " Static Services are ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateStaticServicesScriptFileName: !Ref StaticServicesScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ PrivateDbHost: !Ref DbHost
+ PrivateDbPassword: !Ref DbPassword
+ PrivateDbUser: !Ref DbUser
+ PrivateDbPort: !Ref DbPort
+
+Outputs:
+ # Env parameters
+ PublicSubnetIdDependsOnDNode:
+ Description: Public Subnet Id for distribution node
+ Value: !Ref SubnetId
+ SecurityGroupIDDependsOnDNode:
+ Description: Secrurity Group ID for distribution node
+ Value: !Ref SecurityGroupId
+ KeyPairNameDependsOnDNode:
+ Description: KeyPair for distribution node
+ Value: !Ref EMREc2KeyName
+
+ # Instance parameters
+ IdOfInstance:
+ Description: the id of StaticServices Node Instance
+ Value: !Ref Ec2InstanceOfStaticServicesNode
+ StaticServicesNodePrivateIp:
+ Description: StaticServices Private IP
+ Value: !GetAtt Ec2InstanceOfStaticServicesNode.PrivateIp
+ StaticServicesNodePublicIp:
+ Description: StaticServices Public IP
+ Value: !GetAtt Ec2InstanceOfStaticServicesNode.PublicIp
+ Condition: IsAssociatedPublicIp
diff --git a/cloudformation_templates/ec2-cluster-zk.yaml b/cloudformation_templates/ec2-cluster-zk.yaml
new file mode 100644
index 0000000..7d43e8d
--- /dev/null
+++ b/cloudformation_templates/ec2-cluster-zk.yaml
@@ -0,0 +1,322 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Cluster for ec2 instances and deploy related services, hadoop/spark/zookeeper/hive/jdk/kylin4
+Parameters:
+ # Must passed Parameters
+ SubnetId:
+ Type: String
+ SecurityGroupId:
+ Type: String
+ EMREc2KeyName:
+ Type: String
+ InstanceProfileId:
+ Type: String
+ Description: Must be created at pre-step
+
+# Optional Parameters
+ Ec2Mode:
+ Type: String
+ Default: test
+ AllowedValues:
+ - test
+ - product
+ AssociatedPublicIp:
+ Type: String
+ Default: false
+ AllowedValues:
+ - false
+ - true
+ BucketFullPath:
+ Type: String
+ Description: example s3://xxx/kylin
+ BucketPath:
+ Type: String
+ Description: example /xxx/kylin which Url without prefix s3:/
+ ZookeeperScriptFileName:
+ Type: String
+ Default: prepare-ec2-env-for-zk.sh
+ Ec2InstanceTypeForZookeeper:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.xlarge
+ AllowedValues:
+ - m5.xlarge
+ - m5.2xlarge
+ Ec2InstanceTypeForZookeeperForTest:
+ Description: EC2 instance type
+ Type: String
+ ConstraintDescription: must be a valid EC2 instance type.
+ Default: m5.xlarge
+ AllowedValues:
+ - m5.xlarge
+ Ec2VolumeSizeForZookeeperNode:
+ Type: Number
+ Default: 10
+ MinValue: 10
+ MaxValue: 30
+ Ec2VolumnTypeForZookeeperNode:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+ - gp3
+ - io1
+ - io2
+ - sc1
+ - st1
+ - standard
+ Ec2VolumeSizeForZookeeperNodeForTest:
+ Type: Number
+ Default: 10
+ MinValue: 10
+ MaxValue: 10
+ Ec2VolumnTypeForZookeeperNodeForTest:
+ Type: String
+ Default: gp2
+ AllowedValues:
+ - gp2
+
+Mappings:
+ AWSRegionArch2AMI:
+ cn-north-1:
+ HVMebs: ami-00ac27054b887ff5c
+ cn-northwest-1:
+ HVMebs: ami-0135cb179d33fbe3e
+
+Conditions:
+ NotNullSubnetId:
+ !Not [!Equals [!Ref SubnetId, ""]]
+ NotNullEc2KeyName:
+ !Not [!Equals [!Ref EMREc2KeyName, ""]]
+ IsProductMode: !Equals [!Ref Ec2Mode, "product"]
+ ValidConfigurationForEc2: !And
+ - !Condition NotNullSubnetId
+ - !Condition NotNullEc2KeyName
+ IsAssociatedPublicIp: !Equals [!Ref AssociatedPublicIp, "true"]
+
+Resources:
+ Ec2InstanceOfZookeeperNode01:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Zookeeper Node 1
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref Ec2InstanceTypeForZookeeper
+ - !Ref Ec2InstanceTypeForZookeeperForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Master node in benchmark
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ DeleteOnTermination: true
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForZookeeperNode
+ VolumeType: !Ref Ec2VolumnTypeForZookeeperNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForZookeeperNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForZookeeperNodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateZookeeperScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateZookeeperScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --zk-num ${privateZkNum}
+ echo "Zookeeper is ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateZookeeperScriptFileName: !Ref ZookeeperScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ privateZkNum: 1
+
+ Ec2InstanceOfZookeeperNode02:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Zookeeper Node 2
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref Ec2InstanceTypeForZookeeper
+ - !Ref Ec2InstanceTypeForZookeeperForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Master node in benchmark
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ DeleteOnTermination: true
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForZookeeperNode
+ VolumeType: !Ref Ec2VolumnTypeForZookeeperNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForZookeeperNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForZookeeperNodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateZookeeperScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateZookeeperScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --zk-num ${privateZkNum}
+ echo "Zookeeper is ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateZookeeperScriptFileName: !Ref ZookeeperScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ privateZkNum: 2
+
+ Ec2InstanceOfZookeeperNode03:
+ Type: AWS::EC2::Instance
+ DeletionPolicy: Delete
+ Condition: ValidConfigurationForEc2
+ Properties:
+ ImageId: !FindInMap
+ - AWSRegionArch2AMI
+ - !Ref 'AWS::Region'
+ - HVMebs
+ Tags:
+ - Key: Project
+ Value: Kylin4
+ - Key: Name
+ Value: Zookeeper Node 3
+ InstanceType:
+ !If
+ - IsProductMode
+ - !Ref Ec2InstanceTypeForZookeeper
+ - !Ref Ec2InstanceTypeForZookeeperForTest
+ IamInstanceProfile: !Ref InstanceProfileId
+ NetworkInterfaces:
+ - DeviceIndex: 0
+ Description: Auto create for Master node in benchmark
+ AssociatePublicIpAddress: !Ref AssociatedPublicIp
+ DeleteOnTermination: true
+ SubnetId: !Ref SubnetId
+ GroupSet:
+ - !Ref SecurityGroupId
+ BlockDeviceMappings:
+ - !If
+ - IsProductMode
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForZookeeperNode
+ VolumeType: !Ref Ec2VolumnTypeForZookeeperNode
+ DeleteOnTermination: true
+ - DeviceName: /dev/xvda
+ Ebs:
+ VolumeSize: !Ref Ec2VolumeSizeForZookeeperNodeForTest
+ VolumeType: !Ref Ec2VolumnTypeForZookeeperNodeForTest
+ DeleteOnTermination: true
+ KeyName: !Ref EMREc2KeyName
+ UserData:
+ Fn::Base64:
+ Fn::Sub:
+ - |
+ #!/bin/bash -xe
+ cd /home/ec2-user
+ aws s3 cp ${PrivateBucketFullPath}/scripts/${PrivateZookeeperScriptFileName} . --region ${PrivateRegion}
+ bash ${PrivateZookeeperScriptFileName} --bucket-url ${PrivateBucketPath} --region ${PrivateRegion} --zk-num ${privateZkNum}
+ echo "Zookeeper is ready ..."
+ - PrivateBucketFullPath: !Ref BucketFullPath
+ PrivateZookeeperScriptFileName: !Ref ZookeeperScriptFileName
+ PrivateBucketPath: !Ref BucketPath
+ PrivateRegion: !Ref AWS::Region
+ privateZkNum: 3
+
+Outputs:
+ # Instance 01 parameters
+ IdOfInstanceZookeeper01:
+ Description: the id of Zookeeper Node Instance
+ Value: !Ref Ec2InstanceOfZookeeperNode01
+ ZookeeperNode01PrivateIp:
+ Description: Zookeeper Private IP
+ Value: !GetAtt Ec2InstanceOfZookeeperNode01.PrivateIp
+ ZookeeperNode01PublicIp:
+ Description: Zookeeper Public IP
+ Value: !GetAtt Ec2InstanceOfZookeeperNode01.PublicIp
+ Condition: IsAssociatedPublicIp
+
+ # Instance 02 parameters
+ IdOfInstanceZookeeper02:
+ Description: the id of Zookeeper Node Instance
+ Value: !Ref Ec2InstanceOfZookeeperNode02
+ ZookeeperNode02PrivateIp:
+ Description: Zookeeper Private IP
+ Value: !GetAtt Ec2InstanceOfZookeeperNode02.PrivateIp
+ ZookeeperNode02PublicIp:
+ Description: Zookeeper Public IP
+ Value: !GetAtt Ec2InstanceOfZookeeperNode02.PublicIp
+ Condition: IsAssociatedPublicIp
+
+
+ # Instance 03 parameters
+ IdOfInstanceZookeeper03:
+ Description: the id of Zookeeper Node Instance
+ Value: !Ref Ec2InstanceOfZookeeperNode03
+ ZookeeperNode03PrivateIp:
+ Description: Zookeeper Private IP
+ Value: !GetAtt Ec2InstanceOfZookeeperNode03.PrivateIp
+ ZookeeperNode03PublicIp:
+ Description: Zookeeper Public IP
+ Value: !GetAtt Ec2InstanceOfZookeeperNode03.PublicIp
+ Condition: IsAssociatedPublicIp
diff --git a/cloudformation_templates/ec2-or-emr-vpc.yaml b/cloudformation_templates/ec2-or-emr-vpc.yaml
new file mode 100644
index 0000000..e5c5236
--- /dev/null
+++ b/cloudformation_templates/ec2-or-emr-vpc.yaml
@@ -0,0 +1,283 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+AWSTemplateFormatVersion: "2010-09-09"
+Description: create vpc, subnet and security for emr or ec2 's benchmark
+Parameters:
+ # Must passed parameters
+ # Optional Parameters
+ KylinVersion:
+ Type: String
+ Description: this version is for tag
+ Default: 4.0.0
+ AllowedValues:
+ - 4.0.0
+ ClusterType:
+ Type: String
+ Default: ec2
+ Description: this type is for tag
+ AllowedValues:
+ - ec2
+ VpcCidrBlock:
+ Type: String
+ Default: 10.1.0.0/16
+ SubnetCidrBlock:
+ Type: String
+ Default: 10.1.0.0/24
+ Subnet02CidrBlock:
+ Type: String
+ Default: 10.1.1.0/24
+ Ec2OperationRole:
+ Type: String
+ Description: Should be created by pre-step, must not be null !
+ CidrIp:
+ Type: String
+ Description: An inbound rule permits instances to receive traffic from the specified IPv4 or IPv6 CIDR address range, or from the instances associated with the specified security group.
+
+Conditions:
+ NotNullCidrIp:
+ !Not [!Equals [!Ref CidrIp, ""]]
+
+Mappings:
+ Region2Az:
+ cn-northwest-1:
+ '1': cn-northwest-1a
+ '2': cn-northwest-1b
+
+Resources:
+ Ec2InstanceProfile:
+ Type: AWS::IAM::InstanceProfile
+ DeletionPolicy: Delete
+ Properties:
+ Roles:
+ - !Ref Ec2OperationRole
+ Ec2OrEmrBenchMarkVpc:
+ Type: AWS::EC2::VPC
+ Condition: NotNullCidrIp
+ Properties:
+ CidrBlock: !Ref VpcCidrBlock
+ EnableDnsSupport: true
+ EnableDnsHostnames: true
+ Tags:
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Kylin
+ - !Ref KylinVersion
+ - EmrOrEc2
+ - Vpc
+ DeletionPolicy: Delete
+ Ec2OrEmrBenchMarkSubnet:
+ Type: AWS::EC2::Subnet
+ DeletionPolicy: Delete
+ Properties:
+ VpcId: !Ref Ec2OrEmrBenchMarkVpc
+ CidrBlock: !Ref SubnetCidrBlock
+ AvailabilityZone: !FindInMap
+ - Region2Az
+ - !Ref 'AWS::Region'
+ - '1'
+ Tags:
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Kylin
+ - !Ref KylinVersion
+ - !Ref ClusterType
+ - pub1
+ Ec2OrEmrBenchMarkSubnet02:
+ Type: AWS::EC2::Subnet
+ DeletionPolicy: Delete
+ Properties:
+ VpcId: !Ref Ec2OrEmrBenchMarkVpc
+ CidrBlock: !Ref Subnet02CidrBlock
+ AvailabilityZone: !FindInMap
+ - Region2Az
+ - !Ref 'AWS::Region'
+ - '2'
+ Tags:
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Kylin
+ - !Ref KylinVersion
+ - !Ref ClusterType
+ - pub2
+ Ec2OrEmrBenchMarkDBSubnetGroup:
+ Type: AWS::RDS::DBSubnetGroup
+ DeletionPolicy: Delete
+ Properties:
+ Tags:
+ - Key: Project
+ Value: kylin4
+ - Key: Name
+ Value: subnet group for db instance
+ DBSubnetGroupDescription: subnet for db instance
+ SubnetIds:
+ - !Ref Ec2OrEmrBenchMarkSubnet
+ - !Ref Ec2OrEmrBenchMarkSubnet02
+
+ Ec2OrEmrBenchMarkInternetGateway:
+ DeletionPolicy: Delete
+ Type: AWS::EC2::InternetGateway
+ Properties:
+ Tags:
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Kylin
+ - !Ref KylinVersion
+ - !Ref ClusterType
+ - InternetGateWay
+ AttachGateway:
+ Type: AWS::EC2::VPCGatewayAttachment
+ DeletionPolicy: Delete
+ Properties:
+ VpcId: !Ref Ec2OrEmrBenchMarkVpc
+ InternetGatewayId: !Ref Ec2OrEmrBenchMarkInternetGateway
+ Ec2OrEmrBenchMarkRoutetable:
+ Type: AWS::EC2::RouteTable
+ DeletionPolicy: Delete
+ Properties:
+ VpcId: !Ref Ec2OrEmrBenchMarkVpc
+ Tags:
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Kylin
+ - !Ref KylinVersion
+ - !Ref ClusterType
+ - Routetable
+ Ec2OrEmrBenchMarkRoute:
+ Type: AWS::EC2::Route
+ DependsOn:
+ - Ec2OrEmrBenchMarkInternetGateway
+ - Ec2OrEmrBenchMarkRoutetable
+ DeletionPolicy: Delete
+ Properties:
+ RouteTableId: !Ref Ec2OrEmrBenchMarkRoutetable
+ DestinationCidrBlock: 0.0.0.0/0
+ GatewayId: !Ref Ec2OrEmrBenchMarkInternetGateway
+ SubnetRouteTableAssociation:
+ Type: 'AWS::EC2::SubnetRouteTableAssociation'
+ DeletionPolicy: Delete
+ Properties:
+ SubnetId: !Ref Ec2OrEmrBenchMarkSubnet
+ RouteTableId: !Ref Ec2OrEmrBenchMarkRoutetable
+ Subnet2RouteTableAssociation:
+ Type: 'AWS::EC2::SubnetRouteTableAssociation'
+ DeletionPolicy: Delete
+ Properties:
+ SubnetId: !Ref Ec2OrEmrBenchMarkSubnet02
+ RouteTableId: !Ref Ec2OrEmrBenchMarkRoutetable
+ Ec2OrEmrBenchMarkSecurityGroup:
+ Type: AWS::EC2::SecurityGroup
+ DependsOn: Ec2OrEmrBenchMarkVpc
+ Properties:
+ VpcId: !Ref Ec2OrEmrBenchMarkVpc
+ GroupDescription: Open up SSH access and specify range of ports to itself
+ Tags:
+ - Key: Name
+ Value: !Join
+ - ''
+ - - Kylin
+ - !Ref KylinVersion
+ - !Ref ClusterType
+ - SecurityGroup
+ SecurityGroupIngress:
+ # For SSH
+ - IpProtocol: tcp
+ FromPort: '22'
+ ToPort: '22'
+ CidrIp: !Ref CidrIp
+ # For Grafana
+ - IpProtocol: tcp
+ FromPort: '3000'
+ ToPort: '3000'
+ CidrIp: !Ref CidrIp
+ # For Prometheus
+ - IpProtocol: tcp
+ FromPort: '9090'
+ ToPort: '9100'
+ CidrIp: !Ref CidrIp
+ # For Kylin
+ - IpProtocol: tcp
+ FromPort: '7070'
+ ToPort: '7070'
+ CidrIp: !Ref CidrIp
+ # For Spark Master
+ - IpProtocol: tcp
+ FromPort: '8080'
+ ToPort: '8080'
+ CidrIp: !Ref CidrIp
+ # For Spark Worker/Executor
+ - IpProtocol: tcp
+ FromPort: '4040'
+ ToPort: '4050'
+ CidrIp: !Ref CidrIp
+ DeletionPolicy: Delete
+ Ec2OrEmrBenchMarkSecurityGroupFullTcpIngress:
+ Type: AWS::EC2::SecurityGroupIngress
+ DeletionPolicy: Delete
+ Properties:
+ IpProtocol: tcp
+ SourceSecurityGroupId: !Ref Ec2OrEmrBenchMarkSecurityGroup
+ FromPort: 0
+ ToPort: 65535
+ GroupId: !Ref Ec2OrEmrBenchMarkSecurityGroup
+ Ec2OrEmrBenchMarkSecurityGroupFullUdpIngress:
+ Type: AWS::EC2::SecurityGroupIngress
+ DeletionPolicy: Delete
+ Properties:
+ IpProtocol: udp
+ SourceSecurityGroupId: !Ref Ec2OrEmrBenchMarkSecurityGroup
+ FromPort: 0
+ ToPort: 65535
+ GroupId: !Ref Ec2OrEmrBenchMarkSecurityGroup
+ Ec2OrEmrBenchMarkSecurityGroupFullIcmpIpv4Ingress:
+ Type: AWS::EC2::SecurityGroupIngress
+ DeletionPolicy: Delete
+ Description: special port range from -1 to -1 that means full
+ Properties:
+ IpProtocol: icmp
+ SourceSecurityGroupId: !Ref Ec2OrEmrBenchMarkSecurityGroup
+ FromPort: -1
+ ToPort: -1
+ GroupId: !Ref Ec2OrEmrBenchMarkSecurityGroup
+
+Outputs:
+ VpcId:
+ Description: the id of VPC
+ Value: !Ref Ec2OrEmrBenchMarkVpc
+ SubnetId:
+ Description: the id of subnet
+ Value: !Ref Ec2OrEmrBenchMarkSubnet
+ CidrIpOfVPC:
+ Description: the cidr block ip of VPC
+ Value: !Ref VpcCidrBlock
+ CidrIpOfSubnet:
+ Description: the cidr block ip of subnet
+ Value: !Ref SubnetCidrBlock
+ SecurityGroupId:
+ Description: the id of security group
+ Value: !Ref Ec2OrEmrBenchMarkSecurityGroup
+ InstanceProfileId:
+ Description: Instance profile for cluster
+ Value: !Ref Ec2InstanceProfile
+ SubnetGroupName:
+ Description: Subnet group name for DB instance
+ Value: !Ref Ec2OrEmrBenchMarkDBSubnetGroup
diff --git a/clouds/__init__.py b/clouds/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/clouds/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/clouds/aws.py b/clouds/aws.py
new file mode 100644
index 0000000..d661851
--- /dev/null
+++ b/clouds/aws.py
@@ -0,0 +1,385 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+from ast import literal_eval
+from concurrent.futures import ThreadPoolExecutor
+from itertools import repeat
+from typing import Optional, Dict, List
+
+from constant.config import Config
+from constant.deployment import ScaleType, NodeType
+from constant.kylin_properties_params import KylinProperties
+from constant.path import KYLIN_PROPERTIES_TEMPLATE_DIR
+from constant.yaml_params import Params
+from instances.aws_instance import AWSInstance
+from utils import Utils
+
+logger = logging.getLogger(__name__)
+
+
+class AWS:
+
+ def __init__(self, config) -> None:
+ self.cloud_instance = AWSInstance(config)
+ self.config = config
+
+ @property
+ def is_cluster_ready(self) -> bool:
+ if self.is_instances_ready:
+ return True
+ msg = f'Current cluster is not ready.'
+ logger.warning(msg)
+ return False
+
+ @property
+ def is_prepared_for_scale_cluster(self) -> bool:
+ if self.is_prepared_for_cluster:
+ return True
+ msg = f'Current env for deploying a cluster is not ready.'
+ logger.warning(msg)
+ return False
+
+ @property
+ def is_instances_ready(self) -> bool:
+ return self.cloud_instance.is_ec2_stacks_ready()
+
+ @property
+ def is_prepared_for_cluster(self) -> bool:
+ return self.cloud_instance.is_prepared_for_scale_cluster()
+
+ @property
+ def is_cluster_terminated(self) -> bool:
+ if self.is_instances_terminated:
+ return True
+ msg = 'Current cluster is alive, please destroy cluster first.'
+ logger.warning(msg)
+ return False
+
+ @property
+ def is_instances_terminated(self) -> bool:
+ return self.cloud_instance.is_ec2_stacks_terminated()
+
+ @property
+ def kylin_stack_name(self) -> str:
+ return self.cloud_instance.kylin_stack_name
+
+ @property
+ def is_associated_public_ip(self) -> bool:
+ return self.config[Params.ASSOSICATED_PUBLIC_IP.value] == 'true'
+
+ @property
+ def is_destroy_all(self) -> bool:
+ return self.config[Params.ALWAYS_DESTROY_ALL.value] is True
+
+ def is_target_cluster_ready(self, cluster_num: int) -> bool:
+ if self.is_target_cluster_instances_ready(cluster_num):
+ return True
+ msg = f'Current cluster {cluster_num} is not ready, please deploy the cluster first.'
+ logger.warning(msg)
+ return False
+
+ def is_target_cluster_instances_ready(self, cluster_num: int) -> bool:
+ return self.cloud_instance.is_target_ec2_stacks_ready(cluster_num)
+
+ def validate_params(self) -> None:
+ self.cloud_instance.valid_s3_bucket()
+ self.cloud_instance.valid_iam_role()
+ self.cloud_instance.valid_key_pair()
+ self.cloud_instance.valid_cidr_ip()
+
+ def get_resources(self, stack_name: str) -> Dict:
+ return self.cloud_instance.get_stack_output(stack_name)
+
+ def prepare_for_whole_cluster(self) -> None:
+ self.validate_params()
+ self.cloud_instance.create_vpc_stack()
+ self.cloud_instance.create_rds_stack()
+ self.cloud_instance.create_static_service_stack()
+
+ def init_cluster(self) -> None:
+ if not self.is_instances_ready:
+ self.cloud_instance.create_zk_stack()
+ # Need to refresh its config and start them after created zks
+ self.cloud_instance.after_create_zk_cluster()
+ self.cloud_instance.create_spark_master_stack()
+ self.cloud_instance.create_spark_slave_stack()
+ # Before to create a Kylin node, init local kylin.properties and upload it.
+ self.init_kylin_properties()
+ self.upload_kylin_properties()
+ # # Then create a kylin node
+ self.cloud_instance.create_kylin_stack()
+ self.cloud_instance.start_prometheus_first_time()
+ logger.info('Cluster start successfully.')
+
+ def get_kylin_address(self):
+ kylin_resources = self.get_kylin_resources()
+ kylin4_address = kylin_resources.get(Params.KYLIN4_PRIVATE_IP.value)
+ if self.is_associated_public_ip:
+ kylin4_address = kylin_resources.get(Params.KYLIN4_PUBLIC_IP.value)
+ return kylin4_address
+
+ def get_kylin_resources(self):
+ if not self.is_cluster_ready:
+ self.init_cluster()
+ kylin_resources = self.get_resources(self.kylin_stack_name)
+ return kylin_resources
+
+ def destroy_default_cluster(self):
+ self.terminate_ec2_cluster()
+
+ def terminate_ec2_cluster(self) -> Optional[Dict]:
+ if self.is_cluster_terminated:
+ return
+ self.cloud_instance.terminate_spark_slave_stack()
+ self.cloud_instance.terminate_spark_master_stack()
+ self.cloud_instance.terminate_kylin_stack()
+ self.cloud_instance.terminate_zk_stack()
+ logger.info('Cluster terminated useless nodes.')
+
+ def destroy_rds_and_vpc(self) -> None:
+ if not self.is_destroy_all:
+ return
+ logger.info('Prepare to destroy RDS and VPC and monitor node.')
+ # destroy all will delete rds, please be careful.
+ self.cloud_instance.terminate_static_service_stack()
+ self.cloud_instance.terminate_rds_stack()
+ self.cloud_instance.terminate_vpc_stack()
+ logger.info('Destroy RDS and VPC and monitor node done.')
+
+ def alive_nodes(self):
+ cluster_nums = self.generate_scaled_cluster_nums()
+ self.cloud_instance.alive_nodes(cluster_nums=cluster_nums)
+
+ def scale_up(self, node_type: str, cluster_num: int = None, is_destroy: bool = False) -> None:
+ # validate scale_type & node_type
+ self.validate_node_type(node_type)
+ node_nums = self.generate_scaled_up_list(node_type)
+
+ if node_type == NodeType.KYLIN.value:
+ if self.is_kylin_properties_exists(cluster_num=cluster_num):
+ self.init_kylin_properties(cluster_num=cluster_num)
+ self.upload_kylin_properties(cluster_num=cluster_num)
+
+ exec_pool = ThreadPoolExecutor(max_workers=10)
+ with exec_pool as pool:
+ if node_type == NodeType.KYLIN.value:
+ pool.map(self.cloud_instance.scale_up_kylin, node_nums, repeat(cluster_num, len(node_nums)))
+ else:
+ pool.map(self.cloud_instance.scale_up_worker, node_nums, repeat(cluster_num, len(node_nums)))
+
+ def scale_down(self, node_type: str, cluster_num: int = None, is_destroy: bool = False) -> None:
+ self.validate_node_type(node_type)
+ node_nums = self.generate_scaled_down_list(node_type, is_destroy=is_destroy)
+
+ exec_pool = ThreadPoolExecutor(max_workers=10)
+ with exec_pool as pool:
+ if node_type == NodeType.KYLIN.value:
+ pool.map(self.cloud_instance.scale_down_kylin, node_nums, repeat(cluster_num, len(node_nums)))
+ else:
+ pool.map(self.cloud_instance.scale_down_worker, node_nums, repeat(cluster_num, len(node_nums)))
+
+ def launch_clusters(self, cluster_nums: List[int] = None) -> None:
+ if not cluster_nums:
+ cluster_nums = self.generate_scaled_cluster_nums()
+
+ for num in cluster_nums:
+ self.launch_cluster(cluster_num=num)
+
+ def launch_cluster(self, cluster_num: int) -> None:
+ # Before scale up cluster, check the related kylin.properties must exists.
+ self.validate_cluster(cluster_num=cluster_num)
+ self.is_kylin_properties_exists(cluster_num=cluster_num)
+ self.cloud_instance.scale_up_basic_services_for_cluster(cluster_num=cluster_num)
+ self.init_kylin_properties(cluster_num=cluster_num)
+ self.upload_kylin_properties(cluster_num=cluster_num)
+ self.cloud_instance.scale_up_kylin_for_cluster(cluster_num=cluster_num)
+
+ def validate_cluster(self, cluster_num: int) -> None:
+ cluster_nums = self.generate_scaled_cluster_nums()
+ if cluster_num not in cluster_nums:
+ msg = f'Cluster index: {cluster_num} not in the ' \
+ f'range of {self.config[Config.CLUSTER_INDEXES.value]}, please check.'
+ raise Exception(msg)
+
+ def destroy_clusters(self, cluster_nums: List) -> None:
+ if not cluster_nums:
+ cluster_nums = self.generate_scaled_cluster_nums()
+
+ for num in cluster_nums:
+ self.cloud_instance.destroy_cluster(num)
+
+ def is_kylin_properties_exists_in_clusters(self, cluster_nums: List) -> None:
+ for num in cluster_nums:
+ self.is_kylin_properties_exists(cluster_num=num)
+
+ @staticmethod
+ def is_kylin_properties_exists(cluster_num: int = None, properties_file: str = 'kylin.properties') -> None:
+ dest_path = os.path.join(KYLIN_PROPERTIES_TEMPLATE_DIR.format(
+ cluster_num=cluster_num if cluster_num else 'default'), properties_file)
+ if not os.path.exists(dest_path):
+ raise Exception(f'Expected {dest_path} not exists, please check.')
+
+ def restart_prometheus_server(self) -> None:
+ self.cloud_instance.restart_prometheus_server()
+
+ def after_scale_up(self, node_type: str, cluster_num: int = None) -> None:
+ logger.info(f"Checking exists prometheus config after scale up.")
+ not_exists_nodes = self.cloud_instance.check_prometheus_config_after_scale(node_type, cluster_num)
+ logger.info("Refresh prometheus config after scale up.")
+ if not_exists_nodes:
+ self.cloud_instance.refresh_prometheus_config_after_scale_up(not_exists_nodes)
+ if node_type == NodeType.KYLIN.value:
+ self.cloud_instance.refresh_prometheus_spark_driver_of_kylin_after_scale_up(not_exists_nodes)
+
+ def after_scale_down(self, node_type: str, cluster_num: int = None) -> None:
+ logger.info(f"Checking exists prometheus config after scale down.")
+ exists_nodes = self.cloud_instance.check_prometheus_config_after_scale_down(node_type, cluster_num)
+ logger.info("Refresh prometheus config after scale down.")
+ if exists_nodes:
+ self.cloud_instance.refresh_prometheus_config_after_scale_down(exists_nodes)
+ if node_type == NodeType.KYLIN.value:
+ self.cloud_instance.refresh_prometheus_spark_driver_of_kylin_after_scale_down(exists_nodes)
+
+ def after_launch_clusters(self, cluster_nums: List[int] = None) -> None:
+ # cluster_nums is none means deploy all cluster
+ if not cluster_nums:
+ cluster_nums = self.generate_scaled_cluster_nums()
+
+ logger.info(f"Checking exists prometheus config after launch clusters.")
+ not_exists_nodes = self.cloud_instance.check_prometheus_config_after_launch_clusters(cluster_nums)
+ not_exists_kylin_nodes, not_exists_spark_nodes = self.cloud_instance\
+ .check_spark_metric_config_after_launch_clusters(cluster_nums)
+ logger.info("Refresh prometheus config after launch clusters.")
+ if not_exists_nodes:
+ self.cloud_instance.refresh_prometheus_config_after_launch_clusters(not_exists_nodes)
+ if not_exists_kylin_nodes:
+ self.cloud_instance.refresh_prometheus_spark_metrics_of_kylin_in_cluster_after_scale_up(
+ not_exists_kylin_nodes)
+ if not_exists_spark_nodes:
+ self.cloud_instance.refresh_prometheus_spark_master_in_cluster_after_scale(not_exists_spark_nodes)
+
+ def after_destroy_clusters(self, cluster_nums: List[int] = None) -> None:
+ if not cluster_nums:
+ cluster_nums = self.generate_scaled_cluster_nums()
+ logger.info(f"Checking exists prometheus config after destroy cluster.")
+ exists_nodes = self.cloud_instance.check_prometheus_config_after_destroy_clusters(cluster_nums)
+ exists_kylin_nodes, exists_spark_nodes = self.cloud_instance \
+ .check_spark_metric_config_after_destroy_clusters(cluster_nums)
+ logger.info("Refresh prometheus config after destroy cluster.")
+ if exists_nodes:
+ self.cloud_instance.refresh_prometheus_config_after_destroy_clusters(exists_nodes)
+ if exists_kylin_nodes:
+ self.cloud_instance.refresh_prometheus_config_after_scale_down(exists_kylin_nodes)
+ if exists_spark_nodes:
+ self.cloud_instance.refresh_prometheus_config_after_scale_down(exists_spark_nodes)
+
+ @staticmethod
+ def validate_scale_type(scale_type: str) -> None:
+ if scale_type is None or scale_type not in [e.value for e in ScaleType]:
+ msg = f'Not supported scale type: {scale_type}.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ def validate_scale_range(self) -> None:
+ if not self.cloud_instance.is_valid_scaled_range_of_kylin_nodes:
+ msg = 'Scale down range must be less than scale up range in KYLIN_SCALE_*_NODES.'
+ raise Exception(msg)
+
+ if not self.cloud_instance.is_valid_scaled_range_of_spark_worker_nodes:
+ msg = 'Scale down range must be less than scale up range in SPARK_WORKER_SCALE_*_NODES.'
+ raise Exception(msg)
+
+ @staticmethod
+ def validate_node_type(node_type: str) -> None:
+ if node_type is None or node_type not in [e.value for e in NodeType]:
+ msg = f'Not supported None node type.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ def generate_scaled_up_list(self, node_type: str) -> List:
+ if node_type == NodeType.KYLIN.value:
+ kylin_nodes = literal_eval(self.config[Config.KYLIN_SCALE_UP_NODES.value])
+ return Utils.generate_nodes(kylin_nodes)
+ else:
+ worker_nodes = literal_eval(self.config[Config.SPARK_WORKER_SCALE_UP_NODES.value])
+ return Utils.generate_nodes(worker_nodes)
+
+ def generate_scaled_down_list(self, node_type: str, is_destroy: bool = False) -> List:
+ if node_type == NodeType.KYLIN.value:
+ if is_destroy:
+ scaled_range = self.config[Config.KYLIN_SCALE_UP_NODES.value]
+ else:
+ scaled_range = self.config[Config.KYLIN_SCALE_DOWN_NODES.value]
+ kylin_nodes = literal_eval(scaled_range)
+ return Utils.generate_nodes(kylin_nodes)
+ else:
+ if is_destroy:
+ scaled_range = self.config[Config.SPARK_WORKER_SCALE_UP_NODES.value]
+ else:
+ scaled_range = self.config[Config.SPARK_WORKER_SCALE_DOWN_NODES.value]
+ worker_nodes = literal_eval(scaled_range)
+ return Utils.generate_nodes(worker_nodes)
+
+ def generate_scaled_cluster_nums(self) -> List:
+ cluster_nums = literal_eval(self.config[Config.CLUSTER_INDEXES.value])
+ return Utils.generate_nodes(cluster_nums)
+
+ def validate_s3_bucket(self) -> None:
+ self.cloud_instance.valid_s3_bucket()
+
+ def init_kylin_properties_of_clusters(self, cluster_nums: List) -> None:
+ for num in cluster_nums:
+ self.init_kylin_properties(num)
+
+ def init_kylin_properties(self, cluster_num: int = None) -> None:
+ params: Dict = {
+ # DB params
+ KylinProperties.DB_HOST.value: self.cloud_instance.get_db_host(),
+ KylinProperties.DB_PORT.value: self.cloud_instance.db_port,
+ KylinProperties.DB_USER.value: self.cloud_instance.db_user,
+ KylinProperties.DB_PASSWORD.value: self.cloud_instance.db_password,
+
+ # Zookeeper
+ KylinProperties.ZOOKEEPER_HOST.value: self.cloud_instance.get_target_cluster_zk_host(cluster_num),
+
+ # S3 bucket path
+ KylinProperties.S3_BUCKET_PATH.value: self.cloud_instance.bucket_path,
+
+ # Spark Master
+ KylinProperties.SPARK_MASTER.value: self.cloud_instance.get_target_cluster_spark_master_host(cluster_num),
+ }
+ Utils.render_properties(params=params, cluster_num=cluster_num)
+
+ def upload_needed_files(self, tars: List, jars: List, scripts: List) -> None:
+ self.cloud_instance.upload_tars_to_s3(tars)
+ self.cloud_instance.upload_jars_to_s3(jars)
+ self.cloud_instance.upload_scripts_to_s3(scripts)
+
+ def check_needed_files(self, tars: List, jars: List, scripts: List) -> None:
+ self.cloud_instance.check_tars_on_s3(tars)
+ self.cloud_instance.check_jars_on_s3(jars)
+ self.cloud_instance.check_scripts_on_s3(scripts)
+
+ def upload_kylin_properties_to_clusters(self, cluster_nums: List) -> None:
+ for num in cluster_nums:
+ self.upload_kylin_properties(cluster_num=num)
+
+ def upload_kylin_properties(self, cluster_num: int = None) -> None:
+ # kylin.properties always need to upload
+ self.cloud_instance.upload_kylin_properties_to_s3(cluster_num=cluster_num)
diff --git a/constant/__init__.py b/constant/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/constant/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/constant/client.py b/constant/client.py
new file mode 100644
index 0000000..b8544a6
--- /dev/null
+++ b/constant/client.py
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class Client(Enum):
+ CLOUD_FORMATION = 'cloudformation'
+ EC2 = 'ec2'
+ RDS = 'rds'
diff --git a/constant/commands.py b/constant/commands.py
new file mode 100644
index 0000000..1a2d23b
--- /dev/null
+++ b/constant/commands.py
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class Commands(Enum):
+ STOP_PROMETHEUS_COMMAND = 'lsof -t -i:9090 | xargs -r kill -9'
+ ZKS_CFG_COMMAND = '''echo 'server.1={host1}:2888:3888\nserver.2={host2}:2888:3888\nserver.3={host3}:2888:3888' >> /home/ec2-user/hadoop/zookeeper/conf/zoo.cfg'''
+ # because zk need jdk env which created by root and env config in /root/.bash_profile
+ ZKS_START_COMMAND = 'sudo su && source ~/.bash_profile && ${ZOOKEEPER_HOME}/bin/zkServer.sh start'
+ ZKS_CHECK_CONFIGURED_COMMAND = '''grep -Fq "{host}:2888:3888" /home/ec2-user/hadoop/zookeeper/conf/zoo.cfg; echo $?'''
+ ZKS_CHECK_STARTED_COMMAND = 'sudo lsof -i:2181 >/dev/null 2>&1; echo $?'
+
+ START_PROMETHEUS_COMMAND = 'nohup /home/ec2-user/prometheus/prometheus --config.file=/home/ec2-user/prometheus/prometheus.yml >> /home/ec2-user/prometheus/nohup.out 2>&1 &'
+
+ # because SPARK need jdk env which created by root and env config in /root/.bash_profile
+ START_SPARK_MASTER_COMMAND = 'sudo su && source ~/.bash_profile && ${SPARK_HOME}/sbin/start-master.sh'
+
+ # Prometheus commands
+ PROMETHEUS_CFG_COMMAND = '''echo ' - job_name: "{node}"\n static_configs:\n - targets: ["{host}:9100"]' >> /home/ec2-user/prometheus/prometheus.yml '''
+ SPARK_DRIVER_METRIC_OF_KYLIN_COMMAND = """echo ' - job_name: "kylin-spark-driver-{node}"\n metrics_path: /metrics/prometheus\n static_configs:\n - targets: ["{host}:4040"]' >> /home/ec2-user/prometheus/prometheus.yml """
+ PROMETHEUS_CFG_CHECK_COMMAND = '''grep -Fq "{node}" /home/ec2-user/prometheus/prometheus.yml; echo $?'''
+ PROMETHEUS_DELETE_CFG_COMMAND = '''sed -i "/{node}/,+2d" /home/ec2-user/prometheus/prometheus.yml'''
+ SPARK_DRIVER_METRIC_OF_KYLIN_DELETE_CFG_COMMAND = '''sed -i "/kylin-spark-driver-{node}/,+2d" /home/ec2-user/prometheus/prometheus.yml'''
+
+ # Spark metric commands
+ # Special COMMAND_TEMPLATE for spark metrics into prometheus
+ SPARK_MASTER_METRIC_COMMAND = """echo ' - job_name: "{node}"\n metrics_path: /metrics/master/prometheus\n static_configs:\n - targets: ["{host}:8080"]' >> /home/ec2-user/prometheus/prometheus.yml """
+ SPARK_APPLICATIONS_METRIC_COMMAND = """echo ' - job_name: "{node}"\n metrics_path: /metrics/applications/prometheus\n static_configs:\n - targets: ["{host}:8080"]' >> /home/ec2-user/prometheus/prometheus.yml """
+ SPARK_DRIVER_METRIC_COMMAND = """echo ' - job_name: "{node}"\n metrics_path: /metrics/prometheus\n static_configs:\n - targets: ["{host}:4040"]' >> /home/ec2-user/prometheus/prometheus.yml """
+ SPARK_WORKER_METRIC_COMMAND = """echo ' - job_name: "{node}"\n metrics_path: /metrics/prometheus\n static_configs:\n - targets: ["{host}:4041"]' >> /home/ec2-user/prometheus/prometheus.yml """
+ SPARK_EXECUTORS_METRIC_COMMAND = """echo ' - job_name: "{node}"\n metrics_path: /metrics/executors/prometheus\n static_configs:\n - targets: ["{host}:4040"]' >> /home/ec2-user/prometheus/prometheus.yml """
+
+ SPARK_DECOMMISION_WORKER_COMMAND = 'sudo su && source ~/.bash_profile && ${SPARK_HOME}/sbin/decommission-worker.sh'
diff --git a/constant/config.py b/constant/config.py
new file mode 100644
index 0000000..b0353a7
--- /dev/null
+++ b/constant/config.py
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class Config(Enum):
+ # DEBUG params
+ CLOUD_ADDR = 'CLOUD_ADDR'
+
+ # Other params
+ AWS_REGION = 'AWS_REGION'
+ KYLIN_SCALE_UP_NODES = 'KYLIN_SCALE_UP_NODES'
+ KYLIN_SCALE_DOWN_NODES = 'KYLIN_SCALE_DOWN_NODES'
+ SPARK_WORKER_SCALE_UP_NODES = 'SPARK_WORKER_SCALE_UP_NODES'
+ SPARK_WORKER_SCALE_DOWN_NODES = 'SPARK_WORKER_SCALE_DOWN_NODES'
+ ENABLE_SOFT_AFFINITY = 'ENABLE_LOCAL_CACHE_SOFT_AFFINITY'
+ IAM = 'IAMRole'
+ KEY_PAIR = 'KeyName'
+ CLUSTER_INDEXES = 'CLUSTER_INDEXES'
+ CIDR_IP = 'CIDR_IP'
+
+ # Deploy params
+ DEPLOY_PLATFORM = 'DEPLOY_PLATFORM'
+
+ # Stack Names
+ VPC_STACK = 'VPC_STACK'
+ RDS_STACK = 'RDS_STACK'
+ STATIC_SERVICE_STACK = 'STATIC_SERVICE_STACK'
+ ZOOKEEPERS_STACK = 'ZOOKEEPERS_STACK'
+ KYLIN_STACK = 'KYLIN_STACK'
+ SPARK_MASTER_STACK = 'SPARK_MASTER_STACK'
+ SPARK_WORKER_STACK = 'SPARK_WORKER_STACK'
+
+ # Params
+ EC2_STATIC_SERVICES_PARAMS = 'EC2_STATIC_SERVICES_PARAMS'
+ EC2_KYLIN4_PARAMS = 'EC2_KYLIN4_PARAMS'
+ EC2_KYLIN4_SCALE_PARAMS = 'EC2_KYLIN4_SCALE_PARAMS'
+ EC2_VPC_PARAMS = 'EC2_VPC_PARAMS'
+ EC2_RDS_PARAMS = 'EC2_RDS_PARAMS'
+ EC2_ZOOKEEPERS_PARAMS = 'EC2_ZOOKEEPERS_PARAMS'
+ EC2_SPARK_MASTER_PARAMS = 'EC2_SPARK_MASTER_PARAMS'
+ EC2_SPARK_WORKER_PARAMS = 'EC2_SPARK_WORKER_PARAMS'
+ EC2_SPARK_SCALE_SLAVE_PARAMS = 'EC2_SPARK_SCALE_SLAVE_PARAMS'
+
+ # RDS Params
+ DB_IDENTIFIER = 'DB_IDENTIFIER'
+
+ DB_PORT = 'DB_PORT'
+ DB_USER = 'DB_USER'
+ DB_PASSWORD = 'DB_PASSWORD'
diff --git a/constant/deployment.py b/constant/deployment.py
new file mode 100644
index 0000000..e7ca930
--- /dev/null
+++ b/constant/deployment.py
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class DeployType(Enum):
+ DEPLOY = 'deploy'
+ LIST = 'list'
+ SCALE = 'scale'
+ DESTROY = 'destroy'
+
+
+class ScaleType(Enum):
+ UP = 'up'
+ DOWN = 'down'
+
+
+class NodeType(Enum):
+ KYLIN = 'kylin'
+ SPARK_WORKER = 'spark_worker'
+
+
+class Cluster(Enum):
+ ALL = 'all'
+ DEFAULT = 'default'
diff --git a/constant/kylin_properties_params.py b/constant/kylin_properties_params.py
new file mode 100644
index 0000000..d1e29f1
--- /dev/null
+++ b/constant/kylin_properties_params.py
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class KylinProperties(Enum):
+ DB_HOST = 'DB_HOST'
+ DB_PORT = 'DB_PORT'
+ DB_USER = 'DB_USER'
+ DB_PASSWORD = 'DB_PASSWORD'
+
+ S3_BUCKET_PATH = 'S3_BUCKET_PATH'
+ SPARK_MASTER = 'SPARK_MASTER'
+ ZOOKEEPER_HOST = 'ZOOKEEPER_HOST'
diff --git a/constant/path.py b/constant/path.py
new file mode 100644
index 0000000..833a31f
--- /dev/null
+++ b/constant/path.py
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+CUR_DIR = os.path.dirname(__file__)
+BACKUP_PATH = os.path.join(CUR_DIR, '..', 'backup')
+JARS_PATH = os.path.join(BACKUP_PATH, 'jars')
+SCRIPTS_PATH = os.path.join(BACKUP_PATH, 'scripts')
+TARS_PATH = os.path.join(BACKUP_PATH, 'tars')
+KYLIN_PROPERTIES_DIR = os.path.join(BACKUP_PATH, 'properties')
+
+KYLIN_PROPERTIES_TEMPLATE_DIR = os.path.join(KYLIN_PROPERTIES_DIR, '{cluster_num}')
+PROPERTIES_TEMPLATE_DIR = os.path.join(KYLIN_PROPERTIES_DIR, 'templates')
+
+TEMPLATE_OF_KYLIN_PROPERTIES = 'kylin.properties.template'
+RENDERED_FILE = '.rendered'
+
+ROOT_DIR = os.path.join(CUR_DIR, '..')
+TEMPLATES_PATH = os.path.join(ROOT_DIR, 'cloudformation_templates')
diff --git a/constant/server_mode.py b/constant/server_mode.py
new file mode 100644
index 0000000..e43c37a
--- /dev/null
+++ b/constant/server_mode.py
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class ServerMode(Enum):
+ ALL = 'all'
+ JOB = 'job'
+ QUERY = 'query'
diff --git a/constant/yaml_files.py b/constant/yaml_files.py
new file mode 100644
index 0000000..b0492d7
--- /dev/null
+++ b/constant/yaml_files.py
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class File(Enum):
+ CONFIG_YAML = 'kylin_configs.yaml'
+ VPC_YAML = 'ec2-or-emr-vpc.yaml'
+ RDS_YAML = 'ec2-cluster-rds.yaml'
+ STATIC_SERVICE_YAML = 'ec2-cluster-static-services.yaml'
+ ZOOKEEPERS_SERVICE_YAML = 'ec2-cluster-zk.yaml'
+ KYLIN4_YAML = 'ec2-cluster-kylin4.yaml'
+ KYLIN_SCALE_YAML = 'ec2-cluster-kylin4-template.yaml'
+ SPARK_MASTER_YAML = 'ec2-cluster-spark-master.yaml'
+ SPARK_WORKER_YAML = 'ec2-cluster-spark-slave.yaml'
+ SPARK_WORKER_SCALE_YAML = 'ec2-cluster-spark-slave-template.yaml'
+
+
+class Tar(Enum):
+ KYLIN = 'apache-kylin-{KYLIN_VERSION}-bin-spark3.tar.gz'
+ KYLIN_WITH_SOFT = 'apache-kylin-{KYLIN_VERSION}-bin-spark3-soft.tar.gz'
+ HIVE = 'apache-hive-{HIVE_VERSION}-bin.tar.gz'
+ HADOOP = 'hadoop-{HADOOP_VERSION}.tar.gz'
+ JDK = 'jdk-8u301-linux-x64.tar.gz'
+ NODE = 'node_exporter-{NODE_EXPORTER_VERSION}.linux-amd64.tar.gz'
+ PROMETHEUS = 'prometheus-{PROMETHEUS_VERSION}.linux-amd64.tar.gz'
+ SPARK = 'spark-{SPARK_VERSION}-bin-hadoop{HADOOP_VERSION!s:3.3s}.tgz'
+ ZOOKEEPER = 'zookeeper-{ZOOKEEPER_VERSION}.tar.gz'
diff --git a/constant/yaml_params.py b/constant/yaml_params.py
new file mode 100644
index 0000000..cfb7fcf
--- /dev/null
+++ b/constant/yaml_params.py
@@ -0,0 +1,92 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from enum import Enum
+
+
+class Params(Enum):
+ # global params
+ ASSOSICATED_PUBLIC_IP = 'ASSOSICATED_PUBLIC_IP'
+ ALWAYS_DESTROY_ALL = 'ALWAYS_DESTROY_ALL'
+ S3_URI = 'S3_URI'
+ INSTANCE_ID = 'IdOfInstance'
+ CLUSTER_NUM = 'ClusterNum'
+ CIDR_IP = 'CidrIp'
+ IS_SCALED = 'IsScaled'
+
+ # bucket path of stack
+ BUCKET_FULL_PATH = 'BucketFullPath'
+ BUCKET_PATH = 'BucketPath'
+
+ # Vpc params
+ VPC_ID = 'VpcId'
+ SUBNET_ID = 'SubnetId'
+ SECURITY_GROUP = 'SecurityGroupId'
+ INSTANCE_PROFILE = 'InstanceProfileId'
+ SUBNET_GROUP_NAME = 'SubnetGroupName'
+
+ # Rds params
+ DB_HOST = 'DbHost'
+
+ # Static services params
+ STATIC_SERVICES_PRIVATE_IP = 'StaticServicesNodePrivateIp'
+ STATIC_SERVICES_PUBLIC_IP = 'StaticServicesNodePublicIp'
+ STATIC_SERVICES_NAME = 'StaticServices Node'
+
+ # Zookeeper params
+ ZOOKEEPER_IP = 'ZookeeperNode0{num}PrivateIp'
+ ZOOKEEPER_PUB_IP = 'ZookeeperNode0{num}PublicIp'
+ ZOOKEEPER_INSTANCE_ID = 'IdOfInstanceZookeeper0{num}'
+ ZOOKEEPER_HOSTS = 'ZookeepersHost'
+ # Prometheus params of Zookeeper
+ ZOOKEEPER_NAME = 'Zookeeper_Node_0{num}'
+ ZOOKEEPER_NAME_IN_CLUSTER = 'Zookeeper_Node0{num}_In_Cluster_{cluster}'
+
+ # Kylin 4 params
+ KYLIN4_PUBLIC_IP = 'Kylin4Ec2InstancePublicIp'
+ KYLIN4_PRIVATE_IP = 'Kylin4Ec2InstancePrivateIp'
+
+ # Prometheus params of Kylin 4
+ KYLIN_NAME = 'Kylin_Node'
+ KYLIN_NAME_IN_CLUSTER = 'Kylin_Node_In_Cluster_{cluster}'
+ KYLIN_SPARK_DIRVER_IN_CLUSTER = 'Spark_Driver_Of_Kylin_Node_In_Cluster_{cluster}'
+ KYLIN_SCALE_STACK_NAME = 'ec2-kylin-scale-{num}'
+ KYLIN_SCALE_NODE_NAME_IN_CLUSTER = 'Kylin_Node_Scale_{num}_In_Cluster_{cluster}'
+ KYLIN_SCALE_TARGET_CLUSTER_STACK_NAME = 'ec2-kylin-scale-{num}-in-cluster-{cluster}'
+
+ # Spark master params
+ SPARK_MASTER_HOST = 'SparkMasterNodeHost'
+ SPARK_PUB_IP = 'SparkMasterEc2InstancePublicIp'
+
+ # Prometheus params of Spark master
+ SPARK_MASTER_NAME = 'Spark_Master_Node'
+ SPARK_MASTER_NAME_IN_CLUSTER = 'Spark_Master_Node_In_Cluster_{cluster}'
+ SPARK_MASTER_METRIC_IN_CLUSTER = 'Metric_Of_Spark_Master_Node_In_Cluster_{cluster}'
+
+ # Spark workers params
+ SPARK_WORKER_NUM = 'WorkerNum'
+ SPARK_WORKER_ID = 'IdOfInstanceSlave0{num}'
+ SPARK_WORKER_PRIVATE_IP = 'Slave0{num}Ec2InstancePrivateIp'
+ SPARK_WORKER_PUBLIC_IP = 'Slave0{num}Ec2InstancePublicIp'
+ SPARK_SCALED_WORKER_PRIVATE_IP = 'SlaveEc2InstancePrivateIp'
+ SPARK_SCALED_WORKER_PUBLIC_IP = 'SlaveEc2InstancePublicIp'
+
+ # Prometheus params of Spark workers
+ SPARK_WORKER_NAME = 'Spark_Worker_{num}'
+ SPARK_WORKER_NAME_IN_CLUSTER = 'Spark_Worker_{num}_In_Cluster_{cluster}'
+ SPARK_SCALE_WORKER_NAME_IN_CLUSTER = 'Spark_Worker_Node_Scale_{num}_In_Cluster_{cluster}'
+ SPARK_WORKER_SCALE_TARGET_CLUSTER_STACK_NAME = 'ec2-spark-worker-scale-{num}-in-cluster-{cluster}'
diff --git a/deploy.py b/deploy.py
new file mode 100644
index 0000000..3fdb5e0
--- /dev/null
+++ b/deploy.py
@@ -0,0 +1,98 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import logging.config
+
+from constant.deployment import DeployType, ScaleType, NodeType, Cluster
+
+
+def deploy_on_aws(deploy_type: str, scale_type: str, node_type: str, cluster: str = None) -> None:
+ from engine import Engine
+ aws_engine = Engine()
+ if not aws_engine.is_ec2_cluster:
+ msg = f'Now only supported platform: EC2, please check `DEPLOY_PLATFORM`.'
+ raise Exception(msg)
+
+ if deploy_type == DeployType.DEPLOY.value:
+ # init env for all clusters
+ aws_engine.init_env()
+
+ if cluster == Cluster.ALL.value:
+ aws_engine.launch_default_cluster()
+ aws_engine.launch_all_clusters()
+
+ if not cluster or cluster == Cluster.DEFAULT.value:
+ aws_engine.launch_default_cluster()
+
+ if cluster and cluster.isdigit():
+ aws_engine.launch_cluster(cluster_num=int(cluster))
+
+ elif deploy_type == DeployType.DESTROY.value:
+ if cluster == Cluster.ALL.value:
+ aws_engine.destroy_all_cluster()
+ aws_engine.destroy_default_cluster()
+ aws_engine.refresh_kylin_properties()
+ aws_engine.destroy_rds_and_vpc()
+
+ if not cluster or cluster == Cluster.DEFAULT.value:
+ aws_engine.destroy_default_cluster()
+ aws_engine.refresh_kylin_properties_in_default()
+
+ if cluster and cluster.isdigit():
+ aws_engine.destroy_cluster(cluster_num=int(cluster))
+ aws_engine.refresh_kylin_properties_in_clusters(cluster_nums=[int(cluster)])
+
+ elif deploy_type == DeployType.LIST.value:
+ aws_engine.list_alive_nodes()
+ elif deploy_type == DeployType.SCALE.value:
+ if not scale_type or not node_type:
+ msg = 'Invalid scale params, `scale-type`, `node-type` must be not None.'
+ raise Exception(msg)
+ aws_engine.scale_nodes(scale_type, node_type, cluster)
+
+
+if __name__ == '__main__':
+ logging.config.fileConfig('logging.ini')
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--type", required=False, default=DeployType.LIST.value, dest='type',
+ choices=[e.value for e in DeployType],
+ help="Use 'deploy' to create a cluster or 'destroy' to delete a cluster "
+ "or 'list' to list alive nodes.")
+ parser.add_argument("--scale-type", required=False, dest='scale_type',
+ choices=[e.value for e in ScaleType],
+ help="This param must be used with '--type' and '--node-type' "
+ "Use 'up' to scale up nodes or 'down' to scale down nodes. "
+ "Node type will be in ['kylin', 'spark-worker'].")
+ parser.add_argument("--node-type", required=False, dest='node_type',
+ choices=[e.value for e in NodeType],
+ help="This param must be used with '--type' and '--scale-type' "
+ "Use 'kylin' to scale up/down kylin nodes "
+ "or 'spark-worker' to scale up/down spark_worker nodes. ")
+ # special choices for cluster
+ cluster_choices = [str(num) for num in range(1, 6)]
+ cluster_choices.append('all')
+ cluster_choices.append('default')
+ parser.add_argument("--cluster", required=False, dest='cluster', type=str,
+ choices=cluster_choices,
+ help="Use the `num` in the range `CLUSTER_INDEXES` to specify a cluster index "
+ "which can mark what cluster nodes would be deployed or destroyed."
+ "This param must be used with '--type', "
+ "`default` is for to deploy or destroy nodes of `default` cluster.")
+ args = parser.parse_args()
+ deploy_on_aws(args.type, args.scale_type, args.node_type, args.cluster)
diff --git a/engine.py b/engine.py
new file mode 100644
index 0000000..a8e35d3
--- /dev/null
+++ b/engine.py
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+from typing import List
+
+import yaml
+
+from constant.client import Client
+from constant.config import Config
+from constant.deployment import ScaleType, NodeType, Cluster
+from constant.yaml_files import File
+from engine_utils import EngineUtils
+
+logger = logging.getLogger(__name__)
+
+
+class Engine:
+
+ def __init__(self) -> None:
+ d = os.path.dirname(__file__)
+ with open(os.path.join(d, File.CONFIG_YAML.value)) as stream:
+ config = yaml.safe_load(stream)
+ self.config = config
+ self.is_ec2_cluster = self.config[Config.DEPLOY_PLATFORM.value] == Client.EC2.value
+ self.server_mode = None
+ self.engine_utils = EngineUtils(self.config)
+
+ def launch_default_cluster(self):
+ logger.info('First launch default Kylin Cluster.')
+ self.engine_utils.launch_default_cluster()
+ logger.info('Default Kylin Cluster already start successfully.')
+
+ def destroy_default_cluster(self):
+ logger.info('Start to destroy default Kylin Cluster.')
+ self.engine_utils.destroy_default_cluster()
+ logger.info('Destroy default Kylin Cluster successfully.')
+
+ def destroy_rds_and_vpc(self) -> None:
+ self.engine_utils.destroy_rds_and_vpc()
+
+ def list_alive_nodes(self) -> None:
+ logger.info('Ec2: list alive nodes.')
+ self.engine_utils.alive_nodes()
+ logger.info('Ec2: list alive nodes successfully.')
+
+ def scale_nodes(self, scale_type: str, node_type: str, cluster_num: int = None) -> None:
+ self.engine_utils.validate_scale_range()
+ if cluster_num == Cluster.ALL.value:
+ raise Exception('Current scale nodes not support to scale for `all` clusters.')
+
+ if cluster_num:
+ self.engine_utils.scale_nodes_in_cluster(
+ scale_type=scale_type,
+ node_type=node_type,
+ cluster_num=cluster_num
+ )
+ else:
+ self.engine_utils.scale_nodes(scale_type=scale_type, node_type=node_type)
+
+ if node_type == NodeType.KYLIN.value and scale_type == ScaleType.UP.value:
+ logger.info(f'Note: Current Kylin Node already scaled, please wait a moment to access it.')
+ logger.info(f"Current scaling {scale_type} {node_type} nodes "
+ f"in {cluster_num if cluster_num else 'default'} cluster successfully.")
+
+ def launch_all_clusters(self) -> None:
+ logger.info(f'Current launch other clusters.')
+ self.engine_utils.launch_all_cluster()
+ logger.info(f'Current launch other clusters successfully.')
+
+ def launch_cluster(self, cluster_num: int) -> None:
+ logger.info(f'Current launch cluster {cluster_num}.')
+ self.engine_utils.launch_cluster(cluster_num=cluster_num)
+ logger.info(f'Current launch cluster {cluster_num} successfully.')
+ logger.info(f'Please note that access to Kylin may wait a moment.')
+
+ def destroy_all_cluster(self) -> None:
+ logger.info(f'Current destroy all scaled cluster nodes.')
+ self.engine_utils.destroy_all_cluster()
+ logger.info(f'Current destroy all scaled cluster nodes.')
+
+ def destroy_cluster(self, cluster_num: int) -> None:
+ logger.info(f'Current destroy launch cluster {cluster_num}.')
+ self.engine_utils.destroy_cluster(cluster_num=cluster_num)
+ logger.info(f'Current destroy cluster {cluster_num} successfully.')
+
+ def is_inited_env(self) -> bool:
+ try:
+ self.engine_utils.check_needed_files()
+ return True
+ except AssertionError:
+ return False
+
+ def init_env(self) -> None:
+ self._prepare_files()
+ self._prepare_services()
+
+ def _prepare_files(self) -> None:
+ self.engine_utils.validate_s3_bucket()
+ if self.is_inited_env():
+ logger.info('Env already inited, skip init again.')
+ return
+ self.engine_utils.download_tars()
+ self.engine_utils.download_jars()
+ self.engine_utils.upload_needed_files()
+ # check again
+ assert self.is_inited_env()
+
+ def _prepare_services(self) -> None:
+ # create vpc, rds and monitor node for whole cluster
+ self.engine_utils.prepare_for_cluster()
+
+ def refresh_kylin_properties(self) -> None:
+ self.engine_utils.refresh_kylin_properties()
+
+ def refresh_kylin_properties_in_clusters(self, cluster_nums: List[int]) -> None:
+ self.engine_utils.refresh_kylin_properties_in_clusters(cluster_nums)
+
+ def refresh_kylin_properties_in_default(self) -> None:
+ self.engine_utils.refresh_kylin_properties_in_default()
diff --git a/engine_utils.py b/engine_utils.py
new file mode 100644
index 0000000..823f42a
--- /dev/null
+++ b/engine_utils.py
@@ -0,0 +1,255 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+from typing import List
+
+from clouds.aws import AWS
+from constant.config import Config
+from constant.deployment import NodeType, ScaleType
+from constant.yaml_files import Tar
+from instances.kylin_utils import KylinUtils
+from utils import Utils
+
+logger = logging.getLogger(__name__)
+
+
+class EngineUtils:
+
+ def __init__(self, config):
+ self.aws = AWS(config)
+ self.config = config
+
+ @property
+ def cloud_address(self) -> str:
+ return self.config[Config.CLOUD_ADDR.value]
+
+ def needed_tars(self) -> List:
+ jdk_package = Tar.JDK.value
+ kylin_package = Tar.KYLIN.value.format(KYLIN_VERSION=self.config['KYLIN_VERSION'])
+ if self.config[Config.ENABLE_SOFT_AFFINITY.value] == 'true':
+ kylin_package = Tar.KYLIN_WITH_SOFT.value.format(KYLIN_VERSION=self.config['KYLIN_VERSION'])
+ hive_package = Tar.HIVE.value.format(HIVE_VERSION=self.config['HIVE_VERSION'])
+ hadoop_package = Tar.HADOOP.value.format(HADOOP_VERSION=self.config['HADOOP_VERSION'])
+ node_exporter_package = Tar.NODE.value.format(NODE_EXPORTER_VERSION=self.config['NODE_EXPORTER_VERSION'])
+ prometheus_package = Tar.PROMETHEUS.value.format(PROMETHEUS_VERSION=self.config['PROMETHEUS_VERSION'])
+ spark_package = Tar.SPARK.value.format(SPARK_VERSION=self.config['SPARK_VERSION'],
+ HADOOP_VERSION=self.config['HADOOP_VERSION'])
+ zookeeper_package = Tar.ZOOKEEPER.value.format(ZOOKEEPER_VERSION=self.config['ZOOKEEPER_VERSION'])
+ packages = [jdk_package, kylin_package, hive_package, hadoop_package, node_exporter_package,
+ prometheus_package, spark_package, zookeeper_package]
+ return packages
+
+ def needed_jars(self) -> List:
+ # FIXME: hard version of jars
+ jars = []
+ commons_configuration = 'commons-configuration-1.3.jar'
+ mysql_connector = 'mysql-connector-java-5.1.40.jar'
+ jars.append(commons_configuration)
+ jars.append(mysql_connector)
+ if self.config[Config.ENABLE_SOFT_AFFINITY.value] == 'true':
+ kylin_soft_affinity_cache = 'kylin-soft-affinity-cache-4.0.0-SNAPSHOT.jar'
+ alluxio_client = 'alluxio-2.6.1-client.jar'
+ jars.append(kylin_soft_affinity_cache)
+ jars.append(alluxio_client)
+ return jars
+
+ @staticmethod
+ def needed_scripts() -> List:
+ kylin = 'prepare-ec2-env-for-kylin4.sh'
+ spark_master = 'prepare-ec2-env-for-spark-master.sh'
+ spark_slave = 'prepare-ec2-env-for-spark-slave.sh'
+ static_services = 'prepare-ec2-env-for-static-services.sh'
+ zookeeper = 'prepare-ec2-env-for-zk.sh'
+ return [kylin, spark_master, spark_slave, static_services, zookeeper]
+
+ def launch_default_cluster(self):
+ cloud_addr = self.get_kylin_address()
+ # check kylin status
+ KylinUtils.is_kylin_accessible(cloud_addr)
+
+ def destroy_default_cluster(self):
+ # Before destroy default cluster, scale down all the scaled nodes
+ self.scale_nodes(scale_type=ScaleType.DOWN.value, node_type=NodeType.KYLIN.value)
+ self.scale_nodes(scale_type=ScaleType.DOWN.value, node_type=NodeType.SPARK_WORKER.value)
+
+ self.aws.destroy_default_cluster()
+
+ def alive_nodes(self):
+ self.aws.alive_nodes()
+
+ def scale_nodes(self, scale_type: str, node_type: str) -> None:
+ logger.info(f'Current scaling {scale_type} {node_type} nodes.')
+ self.validate_scale_type(scale_type)
+
+ if scale_type == ScaleType.UP.value:
+ assert self.is_cluster_ready() is True, 'Cluster nodes must be ready.'
+ self.aws.scale_up(node_type=node_type)
+ self.aws.after_scale_up(node_type=node_type)
+
+ elif scale_type == ScaleType.DOWN.value:
+ self.aws.after_scale_down(node_type=node_type)
+ self.aws.scale_down(node_type=node_type)
+
+ self.aws.restart_prometheus_server()
+
+ def scale_nodes_in_cluster(
+ self,
+ scale_type: str,
+ node_type: str,
+ cluster_num: int,
+ is_destroy: bool = False) -> None:
+ logger.info(f"Current scaling {scale_type} {node_type} nodes "
+ f"in cluster {cluster_num if cluster_num else 'default'}.")
+ self.validate_scale_type(scale_type)
+
+ if scale_type == ScaleType.UP.value:
+ assert self.is_target_cluster_ready(cluster_num) is True, 'Cluster nodes must be ready.'
+ self.aws.scale_up(node_type=node_type, cluster_num=cluster_num, is_destroy=is_destroy)
+ self.aws.after_scale_up(node_type=node_type, cluster_num=cluster_num)
+ else:
+ self.aws.after_scale_down(node_type=node_type, cluster_num=cluster_num)
+ self.aws.scale_down(node_type=node_type, cluster_num=cluster_num, is_destroy=is_destroy)
+
+ self.aws.restart_prometheus_server()
+
+ def prepare_for_cluster(self) -> None:
+ # create vpc, rds and monitor node for whole cluster
+ if not self.is_prepared_for_scale_cluster():
+ self.aws.prepare_for_whole_cluster()
+
+ def launch_all_cluster(self) -> None:
+ self.aws.launch_clusters()
+ self.aws.after_launch_clusters()
+ self.aws.restart_prometheus_server()
+
+ def launch_cluster(self, cluster_num: int) -> None:
+ self.aws.launch_clusters(cluster_nums=[cluster_num])
+ self.aws.after_launch_clusters(cluster_nums=[cluster_num])
+ self.aws.restart_prometheus_server()
+
+ def destroy_all_cluster(self) -> None:
+ # Before destroy all nodes, scale down the nodes in the cluster
+ cluster_nums: List[int] = self.aws.generate_scaled_cluster_nums()
+ for num in cluster_nums:
+ self.scale_nodes_in_cluster(
+ scale_type=ScaleType.DOWN.value,
+ node_type=NodeType.KYLIN.value,
+ cluster_num=num, is_destroy=True)
+ self.scale_nodes_in_cluster(
+ scale_type=ScaleType.DOWN.value,
+ node_type=NodeType.SPARK_WORKER.value,
+ cluster_num=num, is_destroy=True)
+
+ self.aws.after_destroy_clusters(cluster_nums=cluster_nums)
+ self.aws.destroy_clusters(cluster_nums=cluster_nums)
+ self.aws.restart_prometheus_server()
+
+ def destroy_cluster(self, cluster_num: int) -> None:
+ self.scale_nodes_in_cluster(
+ scale_type=ScaleType.DOWN.value,
+ node_type=NodeType.KYLIN.value,
+ cluster_num=cluster_num,
+ is_destroy=True)
+
+ self.aws.after_destroy_clusters(cluster_nums=[cluster_num])
+ self.aws.destroy_clusters(cluster_nums=[cluster_num])
+ self.aws.restart_prometheus_server()
+
+ def destroy_rds_and_vpc(self) -> None:
+ self.aws.destroy_rds_and_vpc()
+
+ def is_cluster_ready(self) -> bool:
+ if self.cloud_address:
+ return True
+ return self.aws.is_cluster_ready
+
+ def is_target_cluster_ready(self, cluster_num: int) -> bool:
+ return self.aws.is_target_cluster_instances_ready(cluster_num)
+
+ def is_prepared_for_scale_cluster(self) -> bool:
+ return self.aws.is_prepared_for_scale_cluster
+
+ def get_kylin_address(self) -> str:
+ kylin_address = self.cloud_address
+ if not kylin_address:
+ kylin_address = self.aws.get_kylin_address()
+ assert kylin_address, f'kylin address is None, please check.'
+ return kylin_address
+
+ def download_tars(self) -> None:
+ logger.info("Downloading packages.")
+ packages = self.needed_tars()
+ for package in packages:
+ Utils.download_tar(filename=package)
+
+ def download_jars(self) -> None:
+ logger.info("Downloading jars.")
+ jars = self.needed_jars()
+ for jar in jars:
+ Utils.download_jar(jar)
+ if self.config[Config.ENABLE_SOFT_AFFINITY.value] == 'true':
+ assert Utils.files_in_jars() == 4, f"Needed jars must be 4, not {Utils.files_in_jars()}, " \
+ f"which contains {jars}."
+ else:
+ assert Utils.files_in_jars() == 2, f"Needed jars must be 2, not {Utils.files_in_jars()}, " \
+ f"which contains {jars}."
+
+ def upload_needed_files(self) -> None:
+ logger.info("Start to uploading tars.")
+ self.aws.upload_needed_files(self.needed_tars(), self.needed_jars(), self.needed_scripts())
+ logger.info("Uploaded tars successfully.")
+
+ def check_needed_files(self) -> None:
+ self.aws.check_needed_files(self.needed_tars(), self.needed_jars(), self.needed_scripts())
+
+ def upload_kylin_properties(self, cluster_num: int = None) -> None:
+ logger.info(f"Start to uploading kylin.properties for cluster {cluster_num if cluster_num else 'default'}.")
+ self.aws.upload_kylin_properties(cluster_num=cluster_num)
+ logger.info(f"Uploaded kylin.properties for cluster {cluster_num if cluster_num else 'default'}.")
+
+ def validate_s3_bucket(self) -> None:
+ self.aws.validate_s3_bucket()
+
+ def validate_scale_type(self, scale_type: str) -> None:
+ self.aws.validate_scale_type(scale_type=scale_type)
+
+ def validate_scale_range(self) -> None:
+ self.aws.validate_scale_range()
+
+ def init_kylin_properties(self, cluster_num: int = None) -> None:
+ logger.info(f"Start to init kylin properties for cluster {cluster_num if cluster_num else 'default'}.")
+ self.aws.init_kylin_properties(cluster_num)
+ logger.info(f"Inited kylin properties for cluster {cluster_num if cluster_num else 'default'}.")
+
+ @staticmethod
+ def refresh_kylin_properties() -> None:
+ logger.info('Start to refresh kylin.poperties in `kylin-tpch/properties/default`.')
+ Utils.refresh_kylin_properties()
+ logger.info('Refresh kylin.poperties in `kylin-tpch/properties/default` successfully.')
+
+ @staticmethod
+ def refresh_kylin_properties_in_clusters(cluster_nums: List) -> None:
+ logger.info('Start to refresh kylin.poperties.')
+ Utils.refresh_kylin_properties_in_clusters(cluster_nums=cluster_nums)
+ logger.info('Refresh kylin.poperties successfully.')
+
+ @staticmethod
+ def refresh_kylin_properties_in_default() -> None:
+ logger.info('Start to refresh kylin.poperties.')
+ Utils.refresh_kylin_properties_in_default()
+ logger.info('Refresh kylin.poperties successfully.')
diff --git a/images/accesskey.png b/images/accesskey.png
new file mode 100644
index 0000000..24664ac
Binary files /dev/null and b/images/accesskey.png differ
diff --git a/images/accesskey2.png b/images/accesskey2.png
new file mode 100644
index 0000000..22f7744
Binary files /dev/null and b/images/accesskey2.png differ
diff --git a/images/directory.png b/images/directory.png
new file mode 100644
index 0000000..22ebf22
Binary files /dev/null and b/images/directory.png differ
diff --git a/images/granfana2.png b/images/granfana2.png
new file mode 100644
index 0000000..e25cc99
Binary files /dev/null and b/images/granfana2.png differ
diff --git a/images/granfana3.png b/images/granfana3.png
new file mode 100644
index 0000000..f069ea2
Binary files /dev/null and b/images/granfana3.png differ
diff --git a/images/granfanadashboard.png b/images/granfanadashboard.png
new file mode 100644
index 0000000..26530d4
Binary files /dev/null and b/images/granfanadashboard.png differ
diff --git a/images/granfanadatasource.png b/images/granfanadatasource.png
new file mode 100644
index 0000000..9cc4c6e
Binary files /dev/null and b/images/granfanadatasource.png differ
diff --git a/images/granfanaeditpanel.png b/images/granfanaeditpanel.png
new file mode 100644
index 0000000..3ca9f0e
Binary files /dev/null and b/images/granfanaeditpanel.png differ
diff --git a/images/granfanapanel.png b/images/granfanapanel.png
new file mode 100644
index 0000000..50c7878
Binary files /dev/null and b/images/granfanapanel.png differ
diff --git a/images/jars.png b/images/jars.png
new file mode 100644
index 0000000..4f25645
Binary files /dev/null and b/images/jars.png differ
diff --git a/images/keypair.png b/images/keypair.png
new file mode 100644
index 0000000..4049a0b
Binary files /dev/null and b/images/keypair.png differ
diff --git a/images/kylinproperties.png b/images/kylinproperties.png
new file mode 100644
index 0000000..e246d6a
Binary files /dev/null and b/images/kylinproperties.png differ
diff --git a/images/newpanel.png b/images/newpanel.png
new file mode 100644
index 0000000..324fda4
Binary files /dev/null and b/images/newpanel.png differ
diff --git a/images/prometheusmetrics.png b/images/prometheusmetrics.png
new file mode 100644
index 0000000..595ae61
Binary files /dev/null and b/images/prometheusmetrics.png differ
diff --git a/images/prometheusquery.png b/images/prometheusquery.png
new file mode 100644
index 0000000..23a2ede
Binary files /dev/null and b/images/prometheusquery.png differ
diff --git a/images/renderedkylinproperties.png b/images/renderedkylinproperties.png
new file mode 100644
index 0000000..34f11fb
Binary files /dev/null and b/images/renderedkylinproperties.png differ
diff --git a/images/scripts.png b/images/scripts.png
new file mode 100644
index 0000000..ddf16a4
Binary files /dev/null and b/images/scripts.png differ
diff --git a/images/sparkmetrics.png b/images/sparkmetrics.png
new file mode 100644
index 0000000..7257bd1
Binary files /dev/null and b/images/sparkmetrics.png differ
diff --git a/images/staticserviceparam.png b/images/staticserviceparam.png
new file mode 100644
index 0000000..6461507
Binary files /dev/null and b/images/staticserviceparam.png differ
diff --git a/images/structure.png b/images/structure.png
new file mode 100644
index 0000000..c48b1c0
Binary files /dev/null and b/images/structure.png differ
diff --git a/images/tars.jpg b/images/tars.jpg
new file mode 100644
index 0000000..229db2c
Binary files /dev/null and b/images/tars.jpg differ
diff --git a/images/tars.png b/images/tars.png
new file mode 100644
index 0000000..8019d1b
Binary files /dev/null and b/images/tars.png differ
diff --git a/instances/__init__.py b/instances/__init__.py
new file mode 100644
index 0000000..cce3aca
--- /dev/null
+++ b/instances/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
diff --git a/instances/aws_instance.py b/instances/aws_instance.py
new file mode 100644
index 0000000..cc8b822
--- /dev/null
+++ b/instances/aws_instance.py
@@ -0,0 +1,2163 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+import re
+import time
+from ast import literal_eval
+from typing import Optional, Dict, List, Tuple
+
+import boto3
+import botocore
+from botocore.exceptions import ClientError, WaiterError, ParamValidationError
+
+from constant.client import Client
+from constant.commands import Commands
+from constant.config import Config
+from constant.deployment import NodeType, Cluster
+from constant.path import JARS_PATH, TARS_PATH, SCRIPTS_PATH, KYLIN_PROPERTIES_TEMPLATE_DIR
+from constant.yaml_files import File
+from constant.yaml_params import Params
+from utils import Utils
+
+logger = logging.getLogger(__name__)
+
+
+class AWSInstance:
+
+ def __init__(self, config):
+ self.config = config
+ # global params
+ self.vpc_id = None
+ self.instance_profile = None
+ self.security_group = None
+ self.subnet_group = None
+ self.subnet_id = None
+ self.db_host = None
+
+ @property
+ def is_associated_public_ip(self) -> bool:
+ return self.config[Params.ASSOSICATED_PUBLIC_IP.value] == 'true'
+
+ @property
+ def scaled_spark_workers(self) -> Optional[Tuple]:
+ if self.config[Config.SPARK_WORKER_SCALE_UP_NODES.value]:
+ return literal_eval(self.config[Config.SPARK_WORKER_SCALE_UP_NODES.value])
+ return ()
+
+ @property
+ def scaled_down_spark_workers(self) -> Optional[Tuple]:
+ if self.config[Config.SPARK_WORKER_SCALE_UP_NODES.value]:
+ return literal_eval(self.config[Config.SPARK_WORKER_SCALE_UP_NODES.value])
+ return ()
+
+ @property
+ def scaled_spark_workers_stacks(self) -> List:
+ return self.scaled_target_spark_workers_stacks()
+
+ def scaled_target_spark_workers_stacks(self, cluster_num: int = None) -> List:
+ return [
+ Params.SPARK_WORKER_SCALE_TARGET_CLUSTER_STACK_NAME.value.format(
+ num=i, cluster=cluster_num if cluster_num else 'default')
+ for i in Utils.generate_nodes(self.scaled_spark_workers)]
+
+ @property
+ def scaled_kylin_nodes(self) -> Optional[Tuple]:
+ if self.config[Config.KYLIN_SCALE_UP_NODES.value]:
+ return literal_eval(self.config[Config.KYLIN_SCALE_UP_NODES.value])
+ return ()
+
+ @property
+ def scaled_down_kylin_nodes(self) -> Optional[Tuple]:
+ if self.config[Config.KYLIN_SCALE_UP_NODES.value]:
+ return literal_eval(self.config[Config.KYLIN_SCALE_DOWN_NODES.value])
+ return ()
+
+ @property
+ def scaled_kylin_stacks(self) -> List:
+ return self.scaled_target_kylin_stacks()
+
+ def scaled_target_kylin_stacks(self, cluster_num: int = None) -> List:
+ return [
+ Params.KYLIN_SCALE_TARGET_CLUSTER_STACK_NAME.value.format(
+ num=i, cluster=cluster_num if cluster_num else 'default')
+ for i in Utils.generate_nodes(self.scaled_kylin_nodes)]
+
+ @property
+ def region(self) -> str:
+ return self.config[Config.AWS_REGION.value]
+
+ @property
+ def cf_client(self):
+ return boto3.client(Client.CLOUD_FORMATION.value, region_name=self.region)
+
+ @property
+ def rds_client(self):
+ return boto3.client(Client.RDS.value, region_name=self.region)
+
+ @property
+ def ec2_client(self):
+ return boto3.client(Client.EC2.value, region_name=self.region)
+
+ @property
+ def ssm_client(self):
+ return boto3.client('ssm', region_name=self.region)
+
+ @property
+ def s3_client(self):
+ return boto3.client('s3', region_name=self.region)
+
+ @property
+ def iam_client(self):
+ return boto3.client('iam', region_name=self.region)
+
+ @property
+ def create_complete_waiter(self):
+ return self.cf_client.get_waiter('stack_create_complete')
+
+ @property
+ def delete_complete_waiter(self):
+ return self.cf_client.get_waiter('stack_delete_complete')
+
+ @property
+ def exists_waiter(self):
+ return self.cf_client.get_waiter('stack_exists')
+
+ @property
+ def db_port(self) -> str:
+ return self.config[Config.DB_PORT.value]
+
+ @property
+ def db_user(self) -> str:
+ return self.config[Config.DB_USER.value]
+
+ @property
+ def db_password(self) -> str:
+ return self.config[Config.DB_PASSWORD.value]
+
+ @property
+ def db_identifier(self) -> str:
+ return self.config[Config.DB_IDENTIFIER.value]
+
+ @property
+ def vpc_stack_name(self) -> str:
+ return self.config[Config.VPC_STACK.value]
+
+ @property
+ def rds_stack_name(self) -> str:
+ return self.config[Config.RDS_STACK.value]
+
+ @property
+ def static_service_stack_name(self) -> str:
+ return self.config[Config.STATIC_SERVICE_STACK.value]
+
+ @property
+ def zk_stack_name(self) -> str:
+ return self.config[Config.ZOOKEEPERS_STACK.value]
+
+ @property
+ def kylin_stack_name(self) -> str:
+ return self.config[Config.KYLIN_STACK.value]
+
+ @property
+ def spark_master_stack_name(self) -> str:
+ return self.config[Config.SPARK_MASTER_STACK.value]
+
+ @property
+ def spark_slave_stack_name(self) -> str:
+ return self.config[Config.SPARK_WORKER_STACK.value]
+
+ @property
+ def path_of_vpc_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.VPC_YAML.value)
+
+ @property
+ def path_of_rds_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.RDS_YAML.value)
+
+ @property
+ def path_of_static_service_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.STATIC_SERVICE_YAML.value)
+
+ @property
+ def path_of_zk_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.ZOOKEEPERS_SERVICE_YAML.value)
+
+ @property
+ def path_of_kylin_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.KYLIN4_YAML.value)
+
+ @property
+ def path_of_kylin_scale_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.KYLIN_SCALE_YAML.value)
+
+ @property
+ def path_of_spark_master_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.SPARK_MASTER_YAML.value)
+
+ @property
+ def path_of_spark_slave_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.SPARK_WORKER_YAML.value)
+
+ @property
+ def path_of_spark_slave_scaled_stack(self) -> str:
+ return Utils.full_path_of_yaml(File.SPARK_WORKER_SCALE_YAML.value)
+
+ @property
+ def s3_path_match(self) -> re.Match:
+ original_path: str = self.config[Params.S3_URI.value]
+ match = re.match(pattern=r'^s3://([^/]+)/(.*?([^/]+)/?)$', string=original_path)
+ if not match:
+ raise Exception(f"Invalid S3 bucket path: {original_path}, please check.")
+ return match
+
+ @property
+ def is_valid_scaled_range(self) -> bool:
+ return self.is_valid_scaled_range_of_kylin_nodes and self.is_valid_scaled_range_of_spark_worker_nodes
+
+ @property
+ def is_valid_scaled_range_of_kylin_nodes(self) -> bool:
+ return set(self.scaled_down_kylin_nodes).issubset(self.scaled_kylin_nodes)
+
+ @property
+ def is_valid_scaled_range_of_spark_worker_nodes(self) -> bool:
+ return set(self.scaled_down_spark_workers).issubset(self.scaled_spark_workers)
+
+ @property
+ def bucket_full_path(self) -> str:
+ full_path = self.s3_path_match.group(0)
+ if full_path.endswith('/'):
+ full_path = full_path.rstrip('/')
+ return full_path
+
+ @property
+ def bucket_path(self) -> str:
+ # remove the prefix of 's3:/'
+ path = self.bucket_full_path[len('s3:/'):]
+ return path
+
+ @property
+ def bucket(self) -> str:
+ # get the bucket
+ return self.s3_path_match.group(1)
+
+ @property
+ def bucket_dir(self) -> str:
+ # Note: this dir is for
+ directory = self.s3_path_match.group(2)
+ if directory.endswith('/'):
+ return directory
+ return directory + '/'
+
+ @property
+ def bucket_tars_dir(self) -> str:
+ return self.bucket_dir + 'tar/'
+
+ @property
+ def bucket_scripts_dir(self) -> str:
+ return self.bucket_dir + 'scripts/'
+
+ @property
+ def bucket_kylin_properties_of_default_cluster_dir(self) -> str:
+ return self.bucket_dir + 'properties/default/'
+
+ @property
+ def bucket_kylin_properties_of_other_cluster_dir(self) -> str:
+ return self.bucket_dir + 'properties/{cluster_num}/'
+
+ @property
+ def bucket_jars_dir(self) -> str:
+ return self.bucket_dir + 'jars/'
+
+ @property
+ def iam_role(self) -> str:
+ return self.config[Config.IAM.value]
+
+ @property
+ def key_pair(self) -> str:
+ return self.config[Config.KEY_PAIR.value]
+
+ @property
+ def instance_id_of_static_services(self) -> str:
+ return self.get_instance_id(self.static_service_stack_name)
+
+ @property
+ def cidr_ip(self) -> str:
+ return self.config[Config.CIDR_IP.value]
+
+ def get_vpc_id(self) -> str:
+ if not self.vpc_id:
+ self.vpc_id = self.get_specify_resource_from_output(
+ self.vpc_stack_name, Params.VPC_ID.value
+ )
+ assert self.vpc_id, f'vpc id must not be empty or None.'
+ return self.vpc_id
+
+ def get_instance_profile(self) -> str:
+ if not self.instance_profile:
+ self.instance_profile = self.get_specify_resource_from_output(
+ self.vpc_stack_name, Params.INSTANCE_PROFILE.value
+ )
+ return self.instance_profile
+
+ def get_subnet_id(self) -> str:
+ if not self.subnet_id:
+ self.subnet_id = self.get_specify_resource_from_output(self.vpc_stack_name, Params.SUBNET_ID.value)
+ assert self.subnet_id, 'subnet id must not be empty or None.'
+ return self.subnet_id
+
+ def get_subnet_group(self) -> str:
+ if not self.subnet_group:
+ self.subnet_group = self.get_specify_resource_from_output(
+ self.vpc_stack_name, Params.SUBNET_GROUP_NAME.value
+ )
+ assert self.subnet_group, 'subnet group must not be empty or None.'
+ return self.subnet_group
+
+ def get_db_host(self) -> str:
+ if not self.db_host:
+ self.db_host = self.get_rds_describe()['Endpoint']['Address']
+ assert self.db_host, 'db_host must not be empty or None.'
+ return self.db_host
+
+ def get_security_group_id(self) -> str:
+ if not self.security_group:
+ self.security_group = self.get_specify_resource_from_output(
+ self.vpc_stack_name, Params.SECURITY_GROUP.value
+ )
+ assert self.security_group, f'security_group_id must not be empty or None.'
+ return self.security_group
+
+ def get_instance_id(self, stack_name: str) -> str:
+ if not self.is_stack_complete(stack_name):
+ return ""
+ return self.get_specify_resource_from_output(stack_name, Params.INSTANCE_ID.value)
+
+ def get_instance_ids_of_slaves_stack(self) -> List:
+ return [
+ self.get_specify_resource_from_output(
+ self.spark_slave_stack_name, f'IdOfInstanceOfSlave0{i}'
+ )
+ for i in range(1, 4)
+ ]
+
+ # ============ VPC Services Start ============
+ def create_vpc_stack(self) -> Optional[Dict]:
+ if self.is_stack_complete(self.vpc_stack_name):
+ return
+ params: Dict = self.config[Config.EC2_VPC_PARAMS.value]
+ params[Params.CIDR_IP.value] = self.cidr_ip
+ resp = self.create_stack(
+ stack_name=self.vpc_stack_name,
+ file_path=self.path_of_vpc_stack,
+ params=params,
+ is_capability=True
+ )
+ return resp
+
+ def terminate_vpc_stack(self) -> Optional[Dict]:
+ resp = self.terminate_stack_by_name(self.vpc_stack_name)
+ # Make sure that vpc stack deleted successfully.
+ assert self.is_stack_deleted_complete(self.vpc_stack_name), \
+ f'{self.vpc_stack_name} deleted failed, please check.'
+ return resp
+
+ # ============ VPC Services End ============
+
+ # ============ RDS Services Start ============
+ def get_rds_describe(self) -> Optional[Dict]:
+ if not self.db_identifier:
+ raise Exception(f'{Config.DB_IDENTIFIER.value} must not be empty or None.')
+ if not self.is_rds_exists():
+ return
+ describe_rds: Dict = self.rds_client.describe_db_instances(DBInstanceIdentifier=self.db_identifier)
+ db_instances: List = describe_rds['DBInstances']
+ assert len(db_instances) == 1, f'the identifier of RDS must exist only one.'
+ return db_instances[0]
+
+ def is_rds_exists(self) -> bool:
+ try:
+ self.rds_client.describe_db_instances(DBInstanceIdentifier=self.db_identifier)
+ except self.rds_client.exceptions.DBInstanceNotFoundFault as ex:
+ logger.warning(ex.response['Error']['Message'])
+ logger.info(f'Now creating {self.db_identifier}.')
+ return False
+ return True
+
+ def create_rds_stack(self) -> Optional[Dict]:
+ if self.is_stack_complete(self.rds_stack_name):
+ return
+ if self.is_rds_exists():
+ logger.warning(f'db {self.db_identifier} already exists.')
+ return
+ params: Dict = self.config[Config.EC2_RDS_PARAMS.value]
+ # update needed params
+ params[Params.SUBNET_GROUP_NAME.value] = self.get_subnet_group()
+ params[Params.SECURITY_GROUP.value] = self.get_security_group_id()
+ resp = self.create_stack(
+ stack_name=self.rds_stack_name,
+ file_path=self.path_of_rds_stack,
+ params=params,
+ )
+ # Make sure that rds create successfully.
+ assert self.is_stack_complete(self.rds_stack_name), f'Rds {self.db_identifier} create failed, please check.'
+ return resp
+
+ def terminate_rds_stack(self) -> Optional[Dict]:
+ # Note: terminated rds will not delete it, user can delete db manually.
+ resp = self.terminate_stack_by_name(self.rds_stack_name)
+ return resp
+
+ # ============ RDS Services End ============
+
+ # ============ Static Services Start ============
+ def create_static_service_stack(self) -> Optional[Dict]:
+ if not self.is_rds_ready():
+ msg = f'rds {self.db_identifier} is not ready, please check.'
+ logger.warning(msg)
+ raise Exception(msg)
+
+ if self.is_stack_complete(self.static_service_stack_name):
+ return
+ params: Dict = self.config[Config.EC2_STATIC_SERVICES_PARAMS.value]
+ # update needed params
+ params = self.update_basic_params(params)
+ params[Params.DB_HOST.value] = self.get_db_host()
+
+ resp = self.create_stack(
+ stack_name=self.static_service_stack_name,
+ file_path=self.path_of_static_service_stack,
+ params=params
+ )
+ return resp
+
+ def terminate_static_service_stack(self) -> Optional[Dict]:
+ resp = self.terminate_stack_by_name(self.static_service_stack_name)
+ return resp
+
+ def get_static_services_instance_id(self) -> str:
+ if not self.is_stack_complete(self.static_service_stack_name):
+ return ''
+ return self.get_specify_resource_from_output(self.static_service_stack_name, Params.INSTANCE_ID.value)
+
+ def get_static_services_private_ip(self) -> str:
+ if not self.is_stack_complete(self.static_service_stack_name):
+ return ''
+ return self.get_specify_resource_from_output(
+ self.static_service_stack_name, Params.STATIC_SERVICES_PRIVATE_IP.value)
+
+ def get_static_services_public_ip(self) -> str:
+ if not self.is_stack_complete(self.static_service_stack_name):
+ return ''
+ if not self.is_associated_public_ip:
+ logger.warning('Current static services was associated to a public ip.')
+ return ''
+ return self.get_specify_resource_from_output(
+ self.static_service_stack_name, Params.STATIC_SERVICES_PUBLIC_IP.value)
+
+ def get_static_services_basic_msg(self) -> str:
+ if not self.is_stack_complete(self.static_service_stack_name):
+ return ''
+ res = Params.STATIC_SERVICES_NAME.value + '\t' \
+ + self.instance_id_of_static_services + '\t' \
+ + self.get_static_services_private_ip() + '\t' \
+ + self.get_static_services_public_ip()
+ return res
+
+ # ============ Static Services End ============
+
+ # ============ Zookeeper Services Start ============
+ def get_instance_ids_of_zks_stack(self) -> List:
+ return self.get_instance_ids_of_zks_stack_by_name(self.zk_stack_name)
+
+ def get_instance_ids_of_zks_stack_by_name(self, zk_stack_name: str) -> List:
+ if not self.is_stack_complete(zk_stack_name):
+ return []
+
+ return [
+ self.get_specify_resource_from_output(
+ zk_stack_name, Params.ZOOKEEPER_INSTANCE_ID.value.format(num=i)
+ )
+ for i in range(1, 4)
+ ]
+
+ def get_instance_ips_of_zks_stack(self) -> List:
+ return self.get_instance_ips_of_zks_stack_by_name(self.zk_stack_name)
+
+ def get_instance_ips_of_zks_stack_by_name(self, zk_stack_name: str) -> List:
+ if not self.is_stack_complete(zk_stack_name):
+ return []
+ return [self.get_specify_resource_from_output(
+ zk_stack_name, Params.ZOOKEEPER_IP.value.format(num=i))
+ for i in range(1, 4)
+ ]
+
+ def create_zk_stack(self) -> Optional[Dict]:
+ return self.create_zk_stack_by_cluster()
+
+ def create_zk_stack_by_cluster(self, cluster_num: int = None) -> Optional[Dict]:
+ if cluster_num:
+ zk_stack_name = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ else:
+ zk_stack_name = self.zk_stack_name
+
+ if self.is_stack_complete(zk_stack_name):
+ return
+ params: Dict = self.config[Config.EC2_ZOOKEEPERS_PARAMS.value]
+ # update needed params
+ params = self.update_basic_params(params)
+
+ resp = self.create_stack(
+ stack_name=zk_stack_name,
+ file_path=self.path_of_zk_stack,
+ params=params
+ )
+ assert self.is_stack_complete(zk_stack_name), f'{zk_stack_name} create failed, please check.'
+ return resp
+
+ def terminate_zk_stack(self) -> Optional[Dict]:
+ resp = self.terminate_stack_by_name(self.zk_stack_name)
+ return resp
+
+ def after_create_zk_cluster(self) -> None:
+ self.after_create_zk_of_target_cluster()
+
+ def after_create_zk_of_target_cluster(self, cluster_num: int = None) -> None:
+ if cluster_num:
+ zk_stack = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ else:
+ zk_stack = self.zk_stack_name
+ zk_ips = self.get_instance_ips_of_zks_stack_by_name(zk_stack)
+ zk_ids = self.get_instance_ids_of_zks_stack_by_name(zk_stack)
+ # Check related instances status before refresh zks and start them
+ for zk_id in zk_ids:
+ assert self.is_ec2_instance_running(zk_id), f'Instance {zk_id} is not running, please start it first.'
+
+ # refresh zk cluster cfg, because the zk cfg was not included
+ # FIXME: it's hard code to make sure that zks were already initialized.
+ time.sleep(10)
+
+ self.refresh_zks_cfg(zk_ips=zk_ips, zk_ids=zk_ids)
+ self.start_zks(zk_ids=zk_ids, zk_ips=zk_ips)
+
+ def refresh_zks_cfg(self, zk_ips: List, zk_ids: List) -> None:
+ assert len(zk_ips) == 3, f'Initialized zookeeper ips num is 3, not {len(zk_ips)}.'
+ assert len(zk_ids) == 3, f'Initialized zookeeper ids num is 3, not {len(zk_ids)}.'
+ configured_zks = self.configured_zks_cfg(zk_ids=zk_ids, zk_ips=zk_ips)
+ if self.is_configured_zks(configured_zks):
+ logger.warning('Zookeepers already configured, skip configure.')
+ return
+ need_to_configure_zks = [zk_id for zk_id in zk_ids if zk_id not in configured_zks]
+ refresh_command = Commands.ZKS_CFG_COMMAND.value.format(host1=zk_ips[0], host2=zk_ips[1], host3=zk_ips[2])
+ for zk_id in need_to_configure_zks:
+ self.exec_script_instance_and_return(name_or_id=zk_id, script=refresh_command)
+
+ @staticmethod
+ def is_configured_zks(configured_zks: List) -> bool:
+ return len(configured_zks) == 3
+
+ def configured_zks_cfg(self, zk_ids: List, zk_ips: List) -> List:
+ check_command = Commands.ZKS_CHECK_CONFIGURED_COMMAND.value
+ configured_instances = []
+ for zk_id, zk_ip in zip(zk_ids, zk_ips):
+ resp = self.exec_script_instance_and_return(zk_id, check_command.format(host=zk_ip))
+ if resp['StandardOutputContent'] == '0\n':
+ logger.warning(f'Instance: {zk_id} which ip is {zk_ip} already configured zoo.cfg.')
+ configured_instances.append(zk_id)
+ return configured_instances
+
+ def start_zks(self, zk_ids: List, zk_ips: List) -> None:
+ assert len(zk_ids) == 3, f'Expected to start 3 zookeepers, not {len(zk_ids)}.'
+ started_zks = self.started_zks(zk_ids=zk_ids, zk_ips=zk_ips)
+ if self.is_started_zks(started_zks=started_zks):
+ logger.warning('Zookeepers already started.')
+ return
+
+ need_to_start_zks = [zk_id for zk_id in zk_ids if zk_id not in started_zks]
+ start_zk_command = Commands.ZKS_START_COMMAND.value
+ for zk_id in need_to_start_zks:
+ self.exec_script_instance_and_return(name_or_id=zk_id, script=start_zk_command)
+ logger.info('Start zookeepers successfully.')
+
+ @staticmethod
+ def is_started_zks(started_zks: List) -> bool:
+ return len(started_zks) == 3
+
+ def started_zks(self, zk_ids: List, zk_ips: List) -> List:
+ check_command = Commands.ZKS_CHECK_STARTED_COMMAND.value
+ started_instances = []
+ for zk_id, zk_ip in zip(zk_ids, zk_ips):
+ resp = self.exec_script_instance_and_return(zk_id, check_command)
+ if resp['StandardOutputContent'] == '0\n':
+ logger.warning(f'Instance: {zk_id} which ip is {zk_ip} already started.')
+ started_instances.append(zk_id)
+ return started_instances
+
+ def get_target_cluster_zk_host(self, cluster_num: int = None) -> str:
+ if cluster_num:
+ zk_stack_name = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ else:
+ zk_stack_name = self.zk_stack_name
+ return self.get_zookeepers_host_by_zk_stack_name(zk_stack_name)
+
+ def get_zookeepers_host_by_zk_stack_name(self, zk_stack_name) -> str:
+ zk_ips = self.get_instance_ips_of_zks_stack_by_name(zk_stack_name)
+ res = ','.join([zk_ip + ':2181' for zk_ip in zk_ips])
+ return res
+
+ def get_zks_basic_msg(self) -> List:
+ return self.get_zks_of_target_cluster_msg()
+
+ def get_zks_of_target_cluster_msg(self, cluster_num: int = None) -> List:
+ if cluster_num:
+ zk_stack_name = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ else:
+ zk_stack_name = self.zk_stack_name
+
+ if not self.is_stack_complete(zk_stack_name):
+ return []
+
+ res = [
+ Params.ZOOKEEPER_NAME_IN_CLUSTER.value.format(
+ num=i, cluster=cluster_num if cluster_num else 'default') + '\t'
+ + self.get_specify_resource_from_output(
+ zk_stack_name, Params.ZOOKEEPER_INSTANCE_ID.value.format(num=i)) + '\t'
+ + self.get_specify_resource_from_output(zk_stack_name,
+ Params.ZOOKEEPER_IP.value.format(num=i)) + '\t'
+ + self.get_specify_resource_from_output(zk_stack_name, Params.ZOOKEEPER_PUB_IP.value.format(num=i))
+ for i in range(1, 4)
+ ]
+ return res
+
+ # ============ Zookeeper Services End ============
+
+ # ============ kylin Services Start ============
+ def create_kylin_stack(self) -> Optional[Dict]:
+ return self.create_kylin_stack_by_cluster()
+
+ def create_kylin_stack_by_cluster(self, cluster_num: int = None) -> Optional[Dict]:
+ if cluster_num:
+ kylin_stack_name = self.generate_stack_of_scaled_cluster_for_kylin(cluster_num)
+ zk_stack = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ else:
+ kylin_stack_name = self.kylin_stack_name
+ zk_stack = self.zk_stack_name
+ spark_master_stack = self.spark_master_stack_name
+
+ if self.is_stack_complete(kylin_stack_name):
+ return
+
+ params: Dict = self.config[Config.EC2_KYLIN4_PARAMS.value]
+ params = self.update_basic_params(params)
+ # update extra params
+ params[Params.SPARK_MASTER_HOST.value] = self.get_spark_master_host_by_name(spark_master_stack)
+ params[Params.ZOOKEEPER_HOSTS.value] = self.get_zookeepers_host_by_zk_stack_name(zk_stack)
+ params[Params.DB_HOST.value] = self.get_db_host()
+ params[Params.CLUSTER_NUM.value] = str(cluster_num) if cluster_num else 'default'
+
+ resp = self.create_stack(
+ stack_name=kylin_stack_name,
+ file_path=self.path_of_kylin_stack,
+ params=params
+ )
+ assert self.is_stack_complete(kylin_stack_name), f"Create scaled kylin stack not complete, please check."
+ return resp
+
+ def terminate_kylin_stack(self) -> Optional[Dict]:
+ resp = self.terminate_stack_by_name(self.kylin_stack_name)
+ return resp
+
+ def get_kylin_private_ip(self) -> str:
+ return self.get_kylin_private_ip_of_target_cluster()
+
+ def get_kylin_private_ip_of_target_cluster(self, cluster_num: int = None) -> str:
+ kylin_stack = self.get_kylin_stack_name(cluster_num)
+ return self._get_kylin_private_ip(kylin_stack)
+
+ def get_kylin_private_ip_by_name(self, stack_name: str) -> str:
+ return self._get_kylin_private_ip(stack_name)
+
+ def get_scaled_kylin_private_ip(self, stack_name: str) -> str:
+ return self._get_kylin_private_ip(stack_name)
+
+ def _get_kylin_private_ip(self, stack_name: str) -> str:
+ if not self.is_stack_complete(stack_name):
+ return ""
+ return self.get_specify_resource_from_output(stack_name, Params.KYLIN4_PRIVATE_IP.value)
+
+ def get_kylin_public_ip(self) -> str:
+ return self.get_kylin_public_ip_of_target_cluster()
+
+ def get_kylin_public_ip_of_target_cluster(self, cluster_num: int = None) -> str:
+ kylin_stack = self.get_kylin_stack_name(cluster_num)
+ if not self.is_stack_complete(kylin_stack):
+ return ''
+ if not self.is_associated_public_ip:
+ logger.warning('Current kylin was associated to a public ip.')
+ return ''
+ return self.get_specify_resource_from_output(kylin_stack, Params.KYLIN4_PUBLIC_IP.value)
+
+ def get_kylin_instance_id(self) -> str:
+ return self.get_kylin_instance_id_of_target_cluster()
+
+ def get_kylin_instance_id_of_target_cluster(self, cluster_num: int = None) -> str:
+ kylin_stack = self.get_kylin_stack_name(cluster_num)
+ if not self.is_stack_complete(kylin_stack):
+ return ''
+ return self.get_specify_resource_from_output(kylin_stack, Params.INSTANCE_ID.value)
+
+ def get_kylin_basic_msg(self) -> str:
+ return self.get_kylin_basic_msg_of_target_cluster()
+
+ def get_kylin_basic_msg_of_target_cluster(self, cluster_num: int = None) -> str:
+ if not self.is_stack_complete(self.static_service_stack_name):
+ return ''
+ if not self.is_stack_complete(self.get_kylin_stack_name(cluster_num=cluster_num)):
+ return ''
+ res = Params.KYLIN_NAME_IN_CLUSTER.value.format(cluster=cluster_num if cluster_num else 'default') + '\t' \
+ + self.get_kylin_instance_id_of_target_cluster(cluster_num=cluster_num) + '\t' \
+ + self.get_kylin_private_ip_of_target_cluster(cluster_num=cluster_num) + '\t' \
+ + self.get_kylin_public_ip_of_target_cluster(cluster_num=cluster_num)
+ return res
+
+ def get_kylin_stack_name(self, cluster_num: int = None) -> str:
+ if cluster_num:
+ kylin_stack = self.generate_stack_of_scaled_cluster_for_kylin(cluster_num)
+ else:
+ kylin_stack = self.kylin_stack_name
+
+ return kylin_stack
+
+ def get_scaled_kylin_private_ip(self, stack_name: str) -> str:
+ if not self.is_stack_complete(stack_name):
+ return ''
+ return self.get_specify_resource_from_output(stack_name, Params.KYLIN4_PRIVATE_IP.value)
+
+ def get_scaled_kylin_public_ip(self, stack_name: str) -> str:
+ if not self.is_stack_complete(stack_name):
+ return ""
+ if not self.is_associated_public_ip:
+ logger.warning('Current scaled kylin node was associated to a public ip.')
+ return ''
+ return self.get_specify_resource_from_output(stack_name, Params.KYLIN4_PUBLIC_IP.value)
+
+ def get_scaled_kylin_basic_msg(self) -> List:
+ return self.get_scaled_kylin_basic_msg_of_target_cluster()
+
+ def get_scaled_kylin_basic_msg_of_target_cluster(self, cluster_num: int = None) -> List:
+ msgs = []
+ for stack in self.scaled_target_kylin_stacks(cluster_num):
+ if not self.is_stack_complete(stack):
+ continue
+
+ instance_id = self.get_instance_id(stack)
+ private_ip = self.get_scaled_kylin_private_ip(stack)
+ public_ip = self.get_scaled_kylin_public_ip(stack)
+ msg = stack + '\t' + instance_id + '\t' + private_ip + '\t' + public_ip + '\t'
+ msgs.append(msg)
+ return msgs
+
+ def scale_up_kylin(self, kylin_num: int, cluster_num: int = None) -> Optional[Dict]:
+ """
+ add kylin node
+ """
+
+ if cluster_num:
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num=cluster_num)
+ zk_stack = self.generate_stack_of_scaled_cluster_for_zk(cluster_num=cluster_num)
+ else:
+ spark_master_stack = self.spark_master_stack_name
+ zk_stack = self.zk_stack_name
+
+ if not cluster_num:
+ cluster_num = Cluster.DEFAULT.value
+
+ stack_name = Params.KYLIN_SCALE_TARGET_CLUSTER_STACK_NAME.value.format(
+ num=kylin_num, cluster=cluster_num)
+
+ self._validate_kylin_of_target_cluster_scale(stack_name=stack_name, cluster_num=cluster_num)
+
+ if self.is_stack_complete(stack_name):
+ return
+
+ params: Dict = self.config[Config.EC2_KYLIN4_SCALE_PARAMS.value]
+ # update extra params
+ params = self.update_basic_params(params)
+ params[Params.SPARK_MASTER_HOST.value] = self.get_spark_master_host_by_name(spark_master_stack)
+ params[Params.ZOOKEEPER_HOSTS.value] = self.get_zookeepers_host_by_zk_stack_name(zk_stack)
+ params[Params.DB_HOST.value] = self.get_db_host()
+ params[Params.CLUSTER_NUM.value] = str(cluster_num)
+ params[Params.IS_SCALED.value] = 'true'
+
+ resp = self.create_stack(
+ stack_name=stack_name,
+ file_path=self.path_of_kylin_scale_stack,
+ params=params
+ )
+ assert self.is_stack_complete(stack_name)
+ return resp
+
+ def scale_down_kylin(self, kylin_num: int, cluster_num: int = None) -> Optional[Dict]:
+ if not cluster_num:
+ cluster_num = Cluster.DEFAULT.value
+ stack_name = Params.KYLIN_SCALE_TARGET_CLUSTER_STACK_NAME.value.format(num=kylin_num, cluster=cluster_num)
+ self._validate_kylin_of_target_cluster_scale(stack_name=stack_name, cluster_num=cluster_num)
+
+ # before terminate and delete stack, the worker should be decommissioned.
+ resp = self.terminate_stack_by_name(stack_name)
+
+ assert self.is_stack_deleted_complete(stack_name)
+ return resp
+
+ # ============ kylin Services End ============
+
+ # ============ Spark Master Services Start ============
+ def get_target_cluster_spark_master_host(self, cluster_num: int = None) -> str:
+ if cluster_num:
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ else:
+ spark_master_stack = self.spark_master_stack_name
+
+ return self.get_spark_master_host_by_name(spark_master_stack)
+
+ def get_spark_master_host(self) -> str:
+ return self.get_spark_master_host_of_target_cluster()
+
+ def get_spark_master_host_of_target_cluster(self, cluster_num: int = None) -> str:
+ spark_master_stack = self.get_spark_master_stack_name(cluster_num)
+ return self.get_spark_master_host_by_name(spark_master_stack)
+
+ def get_spark_master_host_by_name(self, spark_master_stack_name: str) -> str:
+ if not self.is_stack_complete(spark_master_stack_name):
+ return ''
+
+ return self.get_specify_resource_from_output(spark_master_stack_name, Params.SPARK_MASTER_HOST.value)
+
+ def get_spark_master_public_ip(self) -> str:
+ return self.get_spark_master_public_ip_of_target_cluster()
+
+ def get_spark_master_public_ip_of_target_cluster(self, cluster_num: int = None) -> str:
+ spark_master_stack = self.get_spark_master_stack_name(cluster_num)
+ if not self.is_stack_complete(spark_master_stack):
+ return ''
+
+ return self.get_specify_resource_from_output(spark_master_stack, Params.SPARK_PUB_IP.value)
+
+ def get_spark_master_stack_name(self, cluster_num: int = None) -> str:
+ if cluster_num:
+ spark_master_stack_name = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ else:
+ spark_master_stack_name = self.spark_master_stack_name
+
+ return spark_master_stack_name
+
+ def create_spark_master_stack(self) -> Optional[Dict]:
+ return self.create_spark_master_stack_by_cluster()
+
+ def create_spark_master_stack_by_cluster(self, cluster_num: int = None) -> Optional[Dict]:
+ if cluster_num:
+ spark_master_stack_name = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ else:
+ spark_master_stack_name = self.spark_master_stack_name
+
+ if self.is_stack_complete(spark_master_stack_name):
+ return
+
+ params: Dict = self.config[Config.EC2_SPARK_MASTER_PARAMS.value]
+ # update needed params
+ params = self.update_basic_params(params)
+ params[Params.DB_HOST.value] = self.get_db_host()
+ resp = self.create_stack(
+ stack_name=spark_master_stack_name,
+ file_path=self.path_of_spark_master_stack,
+ params=params
+ )
+
+ assert self.is_stack_complete(spark_master_stack_name), \
+ f'Current {spark_master_stack_name} stack not create complete, please check.'
+ return resp
+
+ def terminate_spark_master_stack(self) -> Optional[Dict]:
+ resp = self.terminate_stack_by_name(self.spark_master_stack_name)
+ return resp
+
+ def start_spark_master(self) -> None:
+ spark_master_id = self.get_instance_id(self.spark_master_stack_name)
+ if not self.is_ec2_instance_running(spark_master_id):
+ msg = f'Instance of spark master{spark_master_id} was not running, please start it first.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ start_command = Commands.START_SPARK_MASTER_COMMAND.value
+ self.exec_script_instance_and_return(name_or_id=spark_master_id, script=start_command)
+
+ def get_spark_master_instance_id(self) -> str:
+ return self.get_target_cluster_spark_master_host()
+
+ def get_spark_master_instance_id_of_target_cluster(self, cluster_num: int = None) -> str:
+ spark_master_stack = self.get_spark_master_stack_name(cluster_num)
+
+ if not self.is_stack_complete(spark_master_stack):
+ return ''
+
+ return self.get_specify_resource_from_output(spark_master_stack, Params.INSTANCE_ID.value)
+
+ def get_spark_master_msg(self) -> str:
+ return self.get_spark_master_of_target_cluster_msg()
+
+ def get_spark_master_of_target_cluster_msg(self, cluster_num: int = None) -> str:
+ if cluster_num:
+ spark_master_stack_name = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ else:
+ spark_master_stack_name = self.spark_master_stack_name
+
+ if not self.is_stack_complete(spark_master_stack_name):
+ return ''
+
+ res = Params.SPARK_MASTER_NAME_IN_CLUSTER.value.format(
+ cluster=cluster_num if cluster_num else 'default') + '\t' \
+ + self.get_spark_master_instance_id_of_target_cluster(cluster_num) + '\t' \
+ + self.get_spark_master_host_of_target_cluster(cluster_num) + '\t' \
+ + self.get_spark_master_public_ip_of_target_cluster(cluster_num)
+ return res
+
+ # ============ Spark Master Services End ============
+
+ # ============ Spark Slave Services Start ============
+ def create_spark_slave_stack(self) -> Optional:
+ return self.create_spark_slave_stack_by_cluster()
+
+ def create_spark_slave_stack_by_cluster(self, cluster_num: int = None) -> Optional[Dict]:
+ if cluster_num:
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ spark_worker_stack = self.generate_stack_of_scaled_cluster_for_spark_worker(cluster_num)
+ else:
+ spark_master_stack = self.spark_master_stack_name
+ spark_worker_stack = self.spark_slave_stack_name
+
+ if not self.is_stack_complete(spark_master_stack):
+ msg = f'Spark master {spark_master_stack} must be created before create spark slaves.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ if self.is_stack_complete(spark_worker_stack):
+ return
+
+ params: Dict = self.config[Config.EC2_SPARK_WORKER_PARAMS.value]
+ params = self.update_basic_params(params)
+ params[Params.SPARK_MASTER_HOST.value] = self.get_spark_master_host_by_name(spark_master_stack)
+
+ resp = self.create_stack(
+ stack_name=spark_worker_stack,
+ file_path=self.path_of_spark_slave_stack,
+ params=params
+ )
+ return resp
+
+ def terminate_spark_slave_stack(self) -> Optional[Dict]:
+ resp = self.terminate_stack_by_name(self.spark_slave_stack_name)
+ return resp
+
+ def get_instance_ids_of_spark_slave_stack(self) -> Optional[List]:
+ if not self.is_stack_complete(self.spark_slave_stack_name):
+ return
+ return [
+ self.get_specify_resource_from_output(
+ self.spark_slave_stack_name, Params.SPARK_WORKER_ID.value.format(num=i))
+ for i in range(1, 4)
+ ]
+
+ def get_instance_ips_of_spark_slaves_stack(self) -> List:
+ return self.get_instance_ips_of_spark_slaves_stack_by_name(self.spark_slave_stack_name)
+
+ def get_instance_ips_of_spark_slaves_stack_by_name(self, stack_name: str) -> List:
+ if not self.is_stack_complete(stack_name):
+ return []
+ return [
+ self.get_specify_resource_from_output(
+ stack_name, Params.SPARK_WORKER_PRIVATE_IP.value.format(num=i))
+ for i in range(1, 4)
+ ]
+
+ def get_scaled_spark_worker_private_ip(self, stack_name: str) -> str:
+ if not self.is_stack_complete(stack_name):
+ return ''
+ return self.get_specify_resource_from_output(stack_name, Params.SPARK_SCALED_WORKER_PRIVATE_IP.value)
+
+ def get_scaled_spark_worker_public_ip(self, stack_name: str) -> str:
+ if not self.is_stack_complete(stack_name):
+ return ''
+ if not self.is_associated_public_ip:
+ logger.warning('Current spark worker was associated to a public ip.')
+ return ''
+ return self.get_specify_resource_from_output(stack_name, Params.SPARK_SCALED_WORKER_PUBLIC_IP.value)
+
+ def get_spark_slaves_basic_msg(self) -> List:
+ if not self.is_stack_complete(self.spark_slave_stack_name):
+ return []
+
+ res = [
+ Params.SPARK_WORKER_NAME.value.format(num=i) + '\t'
+ + self.get_specify_resource_from_output(
+ self.spark_slave_stack_name, Params.SPARK_WORKER_ID.value.format(num=i)) + '\t'
+ + self.get_specify_resource_from_output(
+ self.spark_slave_stack_name, Params.SPARK_WORKER_PRIVATE_IP.value.format(num=i)) + '\t'
+ + (self.get_specify_resource_from_output(
+ self.spark_slave_stack_name, Params.SPARK_WORKER_PUBLIC_IP.value.format(num=i))
+ if self.is_associated_public_ip else '')
+ for i in range(1, 4)
+ ]
+ return res
+
+ def get_scaled_spark_workers_basic_msg(self) -> List:
+ return self.get_scaled_spark_workers_basic_msg_of_target_cluster()
+
+ def get_spark_workers_of_target_cluster_msg(self, cluster_num: int = None) -> str:
+ if cluster_num:
+ spark_worker_stack_name = self.generate_stack_of_scaled_cluster_for_spark_worker(cluster_num)
+ else:
+ spark_worker_stack_name = self.spark_slave_stack_name
+
+ if not self.is_stack_complete(spark_worker_stack_name):
+ return []
+
+ res = [
+ Params.SPARK_WORKER_NAME_IN_CLUSTER.value.format(num=i, cluster=cluster_num) + '\t'
+ + self.get_specify_resource_from_output(
+ spark_worker_stack_name, Params.SPARK_WORKER_ID.value.format(num=i)) + '\t'
+ + self.get_specify_resource_from_output(
+ spark_worker_stack_name, Params.SPARK_WORKER_PRIVATE_IP.value.format(num=i)) + '\t'
+ + (self.get_specify_resource_from_output(
+ spark_worker_stack_name, Params.SPARK_WORKER_PUBLIC_IP.value.format(num=i))
+ if self.is_associated_public_ip else '')
+ for i in range(1, 4)
+ ]
+ return res
+
+ def get_scaled_spark_workers_basic_msg_of_target_cluster(self, cluster_num: int = None) -> List:
+ msgs = []
+ for stack in self.scaled_target_spark_workers_stacks(cluster_num):
+ if not self.is_stack_complete(stack):
+ continue
+
+ instance_id = self.get_instance_id(stack)
+ private_ip = self.get_scaled_spark_worker_private_ip(stack)
+ public_ip = self.get_scaled_spark_worker_public_ip(stack)
+ msg = stack + '\t' + instance_id + '\t' + private_ip + '\t' + public_ip
+ msgs.append(msg)
+ return msgs
+
+ def scale_up_basic_services_for_cluster(self, cluster_num: int) -> None:
+ # create zookeeper
+ self.create_zk_stack_by_cluster(cluster_num)
+ self.after_create_zk_of_target_cluster(cluster_num)
+
+ # create spark master
+ self.create_spark_master_stack_by_cluster(cluster_num)
+
+ # create spark worker
+ self.create_spark_slave_stack_by_cluster(cluster_num)
+
+ def scale_up_kylin_for_cluster(self, cluster_num: int) -> None:
+ # create kylin
+ self.create_kylin_stack_by_cluster(cluster_num)
+
+ def destroy_cluster(self, cluster_num: int) -> None:
+ scaled_kylin_stack_name = self.generate_stack_of_scaled_cluster_for_kylin(cluster_num)
+ self.terminate_stack_by_name(scaled_kylin_stack_name)
+
+ scaled_spark_slave_stack_name = self.generate_stack_of_scaled_cluster_for_spark_worker(cluster_num)
+ self.terminate_stack_by_name(scaled_spark_slave_stack_name)
+
+ scaled_spark_master_stack_name = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ self.terminate_stack_by_name(scaled_spark_master_stack_name)
+
+ scaled_zk_stack_name = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ self.terminate_stack_by_name(scaled_zk_stack_name)
+
+ def generate_stack_of_scaled_cluster_for_zk(self, cluster_num: int) -> str:
+ return f'{self.zk_stack_name}-scaled-cluster-{cluster_num}'
+
+ def generate_stack_of_scaled_cluster_for_kylin(self, cluster_num: int) -> str:
+ return f'{self.kylin_stack_name}-scaled-cluster-{cluster_num}'
+
+ def generate_stack_of_scaled_cluster_for_spark_master(self, cluster_num: int) -> str:
+ return f'{self.spark_master_stack_name}-scaled-cluster-{cluster_num}'
+
+ def generate_stack_of_scaled_cluster_for_spark_worker(self, cluster_num: int) -> str:
+ return f'{self.spark_slave_stack_name}-scaled-cluster-{cluster_num}'
+
+ def scale_up_worker(self, worker_num: int, cluster_num: int = None) -> Optional[Dict]:
+ """
+ add spark workers for kylin
+ """
+
+ if cluster_num:
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(num=cluster_num)
+ else:
+ spark_master_stack = self.spark_master_stack_name
+
+ if not cluster_num:
+ cluster_num = Cluster.DEFAULT.value
+
+ stack_name = Params.SPARK_WORKER_SCALE_TARGET_CLUSTER_STACK_NAME.value.format(
+ num=worker_num, cluster=cluster_num)
+
+ self._validate_spark_worker_of_target_cluster_scale(stack_name=stack_name, cluster_num=cluster_num)
+
+ if self.is_stack_complete(stack_name):
+ return
+
+ params: Dict = self.config[Config.EC2_SPARK_SCALE_SLAVE_PARAMS.value]
+ params = self.update_basic_params(params)
+ params[Params.SPARK_MASTER_HOST.value] = self.get_spark_master_host_by_name(spark_master_stack)
+ params[Params.SPARK_WORKER_NUM.value] = str(worker_num)
+
+ resp = self.create_stack(
+ stack_name=stack_name,
+ file_path=self.path_of_spark_slave_scaled_stack,
+ params=params
+ )
+ assert self.is_stack_complete(stack_name)
+ return resp
+
+ def scale_down_worker(self, worker_num: int, cluster_num: int = None) -> Optional[Dict]:
+ if not cluster_num:
+ cluster_num = Cluster.DEFAULT.value
+
+ stack_name = Params.SPARK_WORKER_SCALE_TARGET_CLUSTER_STACK_NAME.value.format(
+ num=worker_num, cluster=cluster_num)
+
+ self._validate_spark_worker_of_target_cluster_scale(stack_name=stack_name, cluster_num=cluster_num)
+
+ if self.is_stack_deleted_complete(stack_name):
+ return
+
+ instance_id = self.get_instance_id(stack_name)
+ # spark decommission feature start to be supported in spark 3.1.x.
+ # refer: https://issues.apache.org/jira/browse/SPARK-20624.
+ try:
+ self.exec_script_instance_and_return(
+ name_or_id=instance_id, script=Commands.SPARK_DECOMMISION_WORKER_COMMAND.value)
+ # FIXME: hard code for sleep spark worker to execute remaining jobs
+ # sleep 5 min to ensure all jobs in decommissioned workers are done
+ time.sleep(60 * 3)
+ except AssertionError as ex:
+ logger.error(ex)
+
+ # before terminate and delete stack, the worker should be decommissioned.
+ resp = self.delete_stack(stack_name)
+ assert self.is_stack_deleted_complete(stack_name)
+ return resp
+ # ============ Spark Slave Services End ============
+
+ # ============ Prometheus Services Start ============
+ def start_prometheus_first_time(self) -> None:
+ self.refresh_prometheus_param_map()
+ self.start_prometheus_server()
+
+ def start_prometheus_server(self) -> None:
+ start_command = Commands.START_PROMETHEUS_COMMAND.value
+ instance_id = self.instance_id_of_static_services
+ self.exec_script_instance_and_return(name_or_id=instance_id, script=start_command)
+
+ def stop_prometheus_server(self) -> None:
+ stop_command = Commands.STOP_PROMETHEUS_COMMAND.value
+ instance_id = self.instance_id_of_static_services
+ self.exec_script_instance_and_return(name_or_id=instance_id, script=stop_command)
+
+ def restart_prometheus_server(self) -> None:
+ if not self.is_stack_complete(self.static_service_stack_name):
+ return
+ self.stop_prometheus_server()
+ self.start_prometheus_server()
+
+ def refresh_prometheus_param_map(self) -> None:
+ static_services_id = self.instance_id_of_static_services
+ if not static_services_id:
+ raise Exception(f'Current static services stack was not create complete, please check.')
+
+ refresh_config_commands = self.refresh_prometheus_commands()
+ for command in refresh_config_commands:
+ self.exec_script_instance_and_return(name_or_id=static_services_id, script=command)
+
+ # Special support spark metrics into prometheus
+ spark_config_commands = self.refresh_spark_metrics_commands()
+ for command in spark_config_commands:
+ self.exec_script_instance_and_return(name_or_id=static_services_id, script=command)
+
+ def refresh_prometheus_commands(self) -> List:
+ params = self.prometheus_param_map()
+ # NOTE: the spaces in templates is for prometheus config's indent
+ commands = [Commands.PROMETHEUS_CFG_COMMAND.value.format(node=node, host=host) for node, host in params.items()]
+ return commands
+
+ def after_scale_prometheus_params_map_of_kylin(self, cluster_num: int = None, is_scale_down: bool = False) -> Dict:
+ kylin_ips = self.get_scaled_node_private_ips(cluster_num)[0]
+
+ if is_scale_down:
+ nodes = Utils.generate_nodes(self.scaled_down_kylin_nodes)
+ else:
+ nodes = Utils.generate_nodes(self.scaled_kylin_nodes)
+
+ kylin_node_keys = [Params.KYLIN_SCALE_NODE_NAME_IN_CLUSTER.value.format(
+ num=i, cluster=cluster_num if cluster_num else 'default')
+ for i in nodes]
+
+ kylin_params_map = dict(zip(kylin_node_keys, kylin_ips))
+ return kylin_params_map
+
+ def after_scale_prometheus_params_map_of_spark_worker(
+ self,
+ cluster_num: int = None,
+ is_scale_down: bool = False) -> Dict:
+ spark_workers_ips = self.get_scaled_node_private_ips(cluster_num)[1]
+
+ if is_scale_down:
+ nodes = Utils.generate_nodes(self.scaled_down_spark_workers)
+ else:
+ nodes = Utils.generate_nodes(self.scaled_spark_workers)
+
+ spark_workers_keys = [Params.SPARK_SCALE_WORKER_NAME_IN_CLUSTER.value.format(
+ num=i, cluster=cluster_num if cluster_num else 'default')
+ for i in nodes]
+
+ spark_workers_params_map = dict(zip(spark_workers_keys, spark_workers_ips))
+
+ return spark_workers_params_map
+
+ def after_scale_prometheus_params_map_of_cluster(self, cluster_nums: List[int]) -> Dict:
+ nodes_of_cluster_params_map = self.prometheus_param_map_of_scaled_cluster(cluster_nums)
+ return nodes_of_cluster_params_map
+
+ def after_launch_or_destroy_prometheus_params_map_of_clusters(self, cluster_nums: List[int]) -> Dict:
+ nodes_of_cluster_params_map = self.prometheus_param_map_of_launched_clusters(cluster_nums)
+ return nodes_of_cluster_params_map
+
+ def after_scale_prometheus_params_map_of_cluster_for_spark_metric(self, cluster_nums: List[int]) -> (Dict, Dict):
+ kylin_params_map = self.prometheus_spark_metric_params_of_kylin_in_cluster(cluster_nums)
+ spark_params_map = self.prometheus_spark_metric_params_of_spark_in_cluster(cluster_nums)
+ return kylin_params_map, spark_params_map
+
+ def after_launch_or_destroy_prometheus_params_of_clusters_for_spark_metric(self, cluster_nums: List[int]) -> (Dict, Dict):
+ kylin_params_map = self.prometheus_spark_metric_params_of_kylin_in_cluster(cluster_nums)
+ spark_params_map = self.prometheus_spark_metric_params_of_spark_in_cluster(cluster_nums)
+ return kylin_params_map, spark_params_map
+
+ def is_prometheus_configured(self, host: str) -> bool:
+ static_services_instance_id = self.instance_id_of_static_services
+ check_command = Commands.PROMETHEUS_CFG_CHECK_COMMAND.value.format(node=host)
+ output = self.exec_script_instance_and_return(name_or_id=static_services_instance_id, script=check_command)
+ return output['StandardOutputContent'] == '0\n'
+
+ def get_prometheus_configured_hosts(self, hosts: List) -> List:
+ static_services_instance_id = self.instance_id_of_static_services
+ configured_hosts = []
+ for host in hosts:
+ check_command = Commands.PROMETHEUS_CFG_CHECK_COMMAND.value.format(node=host)
+ output = self.exec_script_instance_and_return(name_or_id=static_services_instance_id, script=check_command)
+ if output['StandardOutputContent'] == '0\n':
+ configured_hosts.append(host)
+ return configured_hosts
+
+ def check_prometheus_config_after_scale(self, node_type: str, cluster_num: int = None) -> Dict:
+ if node_type == NodeType.KYLIN.value:
+ workers_param_map = self.after_scale_prometheus_params_map_of_kylin(cluster_num)
+ else:
+ workers_param_map = self.after_scale_prometheus_params_map_of_spark_worker(cluster_num)
+
+ return self._check_prometheus_not_exists_nodes(workers_param_map)
+
+ def check_prometheus_config_after_scale_cluster(self, cluster_nums: List[int]) -> Dict:
+ params_map = self.after_scale_prometheus_params_map_of_cluster(cluster_nums)
+ return self._check_prometheus_not_exists_nodes(params_map)
+
+ def check_prometheus_config_after_launch_clusters(self, cluster_nums: List[int]) -> Dict:
+ params_map = self.after_launch_or_destroy_prometheus_params_map_of_clusters(cluster_nums=cluster_nums)
+ return self._check_prometheus_not_exists_nodes(params_map)
+
+ def check_spark_metric_config_after_scale_cluster(self, cluster_nums: List[int]) -> (Dict, Dict):
+ kylin_params, spark_params = self.after_scale_prometheus_params_map_of_cluster_for_spark_metric(cluster_nums)
+ not_exists_kylin_nodes: Dict = self._check_prometheus_not_exists_nodes(kylin_params)
+ not_exists_spark_nodes: Dict = self._check_prometheus_not_exists_nodes(spark_params)
+ return not_exists_kylin_nodes, not_exists_spark_nodes
+
+ def check_spark_metric_config_after_launch_clusters(self, cluster_nums: List[int]) -> (Dict, Dict):
+ kylin_params, spark_params = self.after_launch_or_destroy_prometheus_params_of_clusters_for_spark_metric(
+ cluster_nums)
+ not_exists_kylin_nodes: Dict = self._check_prometheus_not_exists_nodes(kylin_params)
+ not_exists_spark_nodes: Dict = self._check_prometheus_not_exists_nodes(spark_params)
+ return not_exists_kylin_nodes, not_exists_spark_nodes
+
+ def check_spark_metric_config_after_scale_down_cluster(self, cluster_nums: List[int]) -> (Dict, Dict):
+ kylin_params, spark_params = self.after_scale_prometheus_params_map_of_cluster_for_spark_metric(cluster_nums)
+ exists_kylin_nodes: Dict = self._check_prometheus_exists_nodes(kylin_params)
+ exists_spark_nodes: Dict = self._check_prometheus_exists_nodes(spark_params)
+ return exists_kylin_nodes, exists_spark_nodes
+
+ def check_spark_metric_config_after_destroy_clusters(self, cluster_nums: List[int]) -> (Dict, Dict):
+ kylin_params, spark_params = self.after_launch_or_destroy_prometheus_params_of_clusters_for_spark_metric(
+ cluster_nums)
+ exists_kylin_nodes: Dict = self._check_prometheus_exists_nodes(kylin_params)
+ exists_spark_nodes: Dict = self._check_prometheus_exists_nodes(spark_params)
+ return exists_kylin_nodes, exists_spark_nodes
+
+ def check_prometheus_config_after_scale_down(self, node_type: str, cluster_num: int = None) -> Dict:
+ if node_type == NodeType.KYLIN.value:
+ workers_param_map = self.after_scale_prometheus_params_map_of_kylin(cluster_num, is_scale_down=True)
+ else:
+ workers_param_map = self.after_scale_prometheus_params_map_of_spark_worker(cluster_num, is_scale_down=True)
+
+ return self._check_prometheus_exists_nodes(params=workers_param_map)
+
+ def _check_prometheus_exists_nodes(self, params: Dict) -> Dict:
+ exists_nodes: Dict = {}
+ for k, v in params.items():
+ command = Commands.PROMETHEUS_CFG_CHECK_COMMAND.value.format(node=k)
+ output = self.exec_script_instance_and_return(
+ name_or_id=self.instance_id_of_static_services, script=command)
+ # output['StandardOutputContent'] = '0\n' means the node exists in the prometheus config
+ if output['StandardOutputContent'] == '0\n':
+ exists_nodes.update({k: v})
+ return exists_nodes
+
+ def _check_prometheus_not_exists_nodes(self, params: Dict) -> Dict:
+ not_exists_nodes: Dict = {}
+ for k, v in params.items():
+ command = Commands.PROMETHEUS_CFG_CHECK_COMMAND.value.format(node=k)
+ output = self.exec_script_instance_and_return(
+ name_or_id=self.instance_id_of_static_services, script=command)
+ # output['StandardOutputContent'] = '1\n' means the node not exists in the prometheus config
+ if output['StandardOutputContent'] == '1\n':
+ not_exists_nodes.update({k: v})
+ return not_exists_nodes
+
+ def check_prometheus_config_after_scale_down_cluster(self, cluster_nums: List[int]) -> Dict:
+ params_map = self.after_scale_prometheus_params_map_of_cluster(cluster_nums)
+ return self._check_prometheus_exists_nodes(params_map)
+
+ def check_prometheus_config_after_destroy_clusters(self, cluster_nums: List[int]) -> Dict:
+ params_map = self.after_launch_or_destroy_prometheus_params_map_of_clusters(cluster_nums)
+ return self._check_prometheus_exists_nodes(params_map)
+
+ def refresh_prometheus_config_after_launch_clusters(self, expected_nodes: Dict) -> None:
+ self.refresh_prometheus_config_after_scale_up(expected_nodes=expected_nodes)
+
+ def refresh_prometheus_config_after_scale_up(self, expected_nodes: Dict) -> None:
+ commands = self.refresh_prometheus_commands_after_scale(expected_nodes)
+ for command in commands:
+ self.exec_script_instance_and_return(name_or_id=self.instance_id_of_static_services, script=command)
+
+ def refresh_prometheus_spark_driver_of_kylin_after_scale_up(self, expected_nodes: Dict) -> None:
+ commands = self.refresh_prometheus_spark_driver_of_kylin_after_scale(expected_nodes)
+ for command in commands:
+ self.exec_script_instance_and_return(name_or_id=self.instance_id_of_static_services, script=command)
+
+ def refresh_prometheus_spark_metrics_of_kylin_in_cluster_after_scale_up(self, expected_nodes: Dict) -> None:
+ commands = self.refresh_prometheus_spark_driver_of_kylin_in_cluster_after_scale(expected_nodes)
+ for command in commands:
+ self.exec_script_instance_and_return(name_or_id=self.instance_id_of_static_services, script=command)
+
+ def refresh_prometheus_config_after_destroy_clusters(self, exists_nodes: Dict) -> None:
+ self.refresh_prometheus_config_after_scale_down(exists_nodes)
+
+ def refresh_prometheus_config_after_scale_down(self, exists_nodes: Dict) -> None:
+ commands = [Commands.PROMETHEUS_DELETE_CFG_COMMAND.value.format(node=worker) for worker in exists_nodes.keys()]
+ for command in commands:
+ self.exec_script_instance_and_return(name_or_id=self.instance_id_of_static_services, script=command)
+
+ def refresh_prometheus_spark_driver_of_kylin_after_scale_down(self, exists_nodes: Dict) -> None:
+ commands = [Commands.SPARK_DRIVER_METRIC_OF_KYLIN_DELETE_CFG_COMMAND.value.format(node=worker)
+ for worker in exists_nodes.keys()]
+ for command in commands:
+ self.exec_script_instance_and_return(name_or_id=self.instance_id_of_static_services, script=command)
+
+ # ============ Prometheus Services End ============
+
+ # ============ Utils Services Start ============
+ def is_scaled_stacks_in_clusters_terminated(self, cluster_nums: List) -> bool:
+ for num in cluster_nums:
+ for stack in self.scaled_target_spark_workers_stacks():
+ if not self.is_stack_deleted_complete(stack):
+ logger.warning(f'Scaled Spark Workers Stack:{stack} '
+ f'in Cluster {num} was not terminated, please check.')
+ return False
+ for stack in self.scaled_target_kylin_stacks(num):
+ if not self.is_stack_deleted_complete(stack):
+ logger.warning(f'Scaled Kylin Stack:{stack} '
+ f'in Cluster {num} was not terminated, please check.')
+ return False
+ return True
+
+ def is_target_clusters_terminated(self, cluster_nums: List) -> bool:
+ for num in cluster_nums:
+ return self.is_target_cluster_terminated(num)
+
+ def is_target_cluster_terminated(self, cluster_num: int) -> bool:
+ zk_stack = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+
+ if not self.is_stack_deleted_complete(zk_stack):
+ logger.warning(f'Stack {zk_stack} in cluster {cluster_num} was not terminated, please check.')
+ return False
+
+ kylin_stack = self.generate_stack_of_scaled_cluster_for_kylin(cluster_num)
+ if not self.is_stack_deleted_complete(kylin_stack):
+ logger.warning(f'Stack {kylin_stack} in cluster {cluster_num} was not terminated, please check.')
+ return False
+
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ if not self.is_stack_deleted_complete(spark_master_stack):
+ logger.warning(f'Stack {spark_master_stack} in cluster {cluster_num} was not terminated, please check.')
+ return False
+
+ spark_workers_stack = self.generate_stack_of_scaled_cluster_for_spark_worker(cluster_num)
+ if not self.is_stack_deleted_complete(spark_workers_stack):
+ logger.warning(f'Stack {spark_workers_stack} in cluster {cluster_num} was not terminated, please check.')
+ return False
+
+ return True
+
+
+ def update_basic_params(self, params: Dict) -> Dict:
+ params[Params.SUBNET_ID.value] = self.get_subnet_id()
+ params[Params.SECURITY_GROUP.value] = self.get_security_group_id()
+ params[Params.INSTANCE_PROFILE.value] = self.get_instance_profile()
+ # update bucket path
+ params[Params.BUCKET_FULL_PATH.value] = self.bucket_full_path
+ params[Params.BUCKET_PATH.value] = self.bucket_path
+ return params
+
+ def terminate_stack_by_name(self, stack_name: str) -> Optional[Dict]:
+ if self.is_stack_deleted_complete(stack_name):
+ return
+
+ resp = self.delete_stack(stack_name)
+ assert self.is_stack_deleted_complete(stack_name), f'Current delete stack {stack_name} failed, please check.'
+ return resp
+
+ def create_stack(self, stack_name: str, file_path: str, params: Dict, is_capability: bool = False) -> Dict:
+ try:
+ if is_capability:
+ resp = self.cf_client.create_stack(
+ StackName=stack_name,
+ TemplateBody=Utils.read_template(file_path),
+ Parameters=[{'ParameterKey': k, 'ParameterValue': v} for k, v in params.items()],
+ Capabilities=['CAPABILITY_IAM']
+ )
+ else:
+ resp = self.cf_client.create_stack(
+ StackName=stack_name,
+ TemplateBody=Utils.read_template(file_path),
+ Parameters=[{'ParameterKey': k, 'ParameterValue': v} for k, v in params.items()],
+ )
+ return resp
+ except ParamValidationError as ex:
+ logger.error(ex)
+ assert self.is_stack_complete(stack_name=stack_name), \
+ f"Stack {stack_name} not create complete, please check."
+
+ def delete_stack(self, stack_name: str) -> Dict:
+ logger.info(f'Current terminating stack: {stack_name}.')
+ resp = self.cf_client.delete_stack(StackName=stack_name)
+ return resp
+
+ def refresh_spark_metrics_commands(self) -> List:
+ spark_master_host = self.get_spark_master_host()
+ # kylin ip will be the host of spark driver and spark worker
+ kylin_ip = self.get_kylin_private_ip()
+ commands = [
+ Commands.SPARK_DRIVER_METRIC_COMMAND.value.format(node='spark_driver_in_kylin', host=kylin_ip),
+ Commands.SPARK_WORKER_METRIC_COMMAND.value.format(node='spark_worker_in_kylin', host=kylin_ip),
+ Commands.SPARK_APPLICATIONS_METRIC_COMMAND.value.format(node='spark_applications', host=spark_master_host),
+ Commands.SPARK_MASTER_METRIC_COMMAND.value.format(node='spark_master', host=spark_master_host),
+ Commands.SPARK_EXECUTORS_METRIC_COMMAND.value.format(node='spark_executors', host=spark_master_host),
+ ]
+ return commands
+
+ @staticmethod
+ def refresh_prometheus_commands_after_scale(expected_nodes: Dict) -> List:
+ commands = [Commands.PROMETHEUS_CFG_COMMAND.value.format(node=node, host=host)
+ for node, host in expected_nodes.items()]
+ return commands
+
+ @staticmethod
+ def refresh_prometheus_spark_driver_of_kylin_after_scale(expected_nodes: Dict) -> List:
+ commands = [Commands.SPARK_DRIVER_METRIC_OF_KYLIN_COMMAND.value.format(node=node, host=host)
+ for node, host in expected_nodes.items()]
+ return commands
+
+ @staticmethod
+ def refresh_prometheus_spark_driver_of_kylin_in_cluster_after_scale(expected_nodes: Dict) -> List:
+ commands = [Commands.SPARK_DRIVER_METRIC_COMMAND.value.format(node=node, host=host)
+ for node, host in expected_nodes.items()]
+ return commands
+
+ @staticmethod
+ def refresh_prometheus_spark_executor_in_cluster_after_scale(expected_nodes: Dict) -> List:
+ commands = [Commands.SPARK_EXECUTORS_METRIC_COMMAND.value.format(node=node, host=host)
+ for node, host in expected_nodes.items()]
+ return commands
+
+ @staticmethod
+ def refresh_prometheus_spark_master_in_cluster_after_scale(expected_nodes: Dict) -> List:
+ commands = [Commands.SPARK_MASTER_METRIC_COMMAND.value.format(node=node, host=host)
+ for node, host in expected_nodes.items()]
+ return commands
+
+ @staticmethod
+ def refresh_prometheus_spark_application_in_cluster_after_scale(expected_nodes: Dict) -> List:
+ commands = [Commands.SPARK_APPLICATIONS_METRIC_COMMAND.value.format(node=node, host=host)
+ for node, host in expected_nodes.items()]
+ return commands
+
+ def prometheus_param_map(self) -> Dict:
+ params_map: Dict = {}
+ static_ip = self.get_static_services_private_ip()
+ static_map = {Params.STATIC_SERVICES_NAME.value: static_ip}
+
+ kylin_ip = self.get_kylin_private_ip()
+ kylin_map = {Params.KYLIN_NAME.value: kylin_ip}
+
+ spark_master_ip = self.get_spark_master_host()
+ spark_master_map = {Params.SPARK_MASTER_NAME.value: spark_master_ip}
+
+ zk_ips = self.get_instance_ips_of_zks_stack()
+ zks_map = {
+ Params.ZOOKEEPER_NAME.value.format(num=i): zk_ips[i - 1] for i in range(1, 4)
+ }
+ spark_slaves_ips = self.get_instance_ips_of_spark_slaves_stack()
+ spark_slaves_map = {
+ Params.SPARK_WORKER_NAME.value.format(num=i): spark_slaves_ips[i - 1] for i in range(1, 4)
+ }
+ params_map.update(static_map)
+ params_map.update(kylin_map)
+ params_map.update(spark_master_map)
+ params_map.update(zks_map)
+ params_map.update(spark_slaves_map)
+
+ return params_map
+
+ def prometheus_param_map_of_scaled_cluster(self, cluster_nums: List[int]) -> Dict:
+ params_map: Dict = {}
+ for num in cluster_nums:
+ # kylin stack
+ kylin_map = self.kylin_param_map_by_cluster_num(num)
+ if kylin_map:
+ params_map.update(kylin_map)
+
+ # spark master
+ spark_master_map = self.spark_master_param_map_by_num(num)
+ if spark_master_map:
+ params_map.update(spark_master_map)
+
+ # zookeeper
+ zks_map = self.zks_param_map_by_num(num)
+ if zks_map:
+ params_map.update(zks_map)
+
+ # spark worker
+ spark_slaves_map = self.spark_worker_param_map(num)
+ if not spark_slaves_map:
+ params_map.update(spark_slaves_map)
+
+ return params_map
+
+ def prometheus_param_map_of_launched_clusters(self, cluster_nums: List[int]) -> Dict:
+ params_map: Dict = {}
+ for num in cluster_nums:
+ param: Dict = self.prometheus_param_map_of_launched_cluster(cluster_num=num)
+ params_map.update(param)
+
+ return params_map
+
+ def prometheus_param_map_of_launched_cluster(self, cluster_num: int) -> Dict:
+ params_map: Dict = {}
+ # kylin stack
+ kylin_map = self.kylin_param_map_by_cluster_num(num=cluster_num)
+ if kylin_map:
+ params_map.update(kylin_map)
+
+ # spark master
+ spark_master_map = self.spark_master_param_map_by_num(num=cluster_num)
+ if spark_master_map:
+ params_map.update(spark_master_map)
+
+ # zookeeper
+ zks_map = self.zks_param_map_by_num(num=cluster_num)
+ if zks_map:
+ params_map.update(zks_map)
+
+ # spark worker
+ spark_slaves_map = self.spark_worker_param_map(num=cluster_num)
+ if not spark_slaves_map:
+ params_map.update(spark_slaves_map)
+
+ return params_map
+
+ def prometheus_spark_metric_params_of_kylin_in_cluster(self, cluster_nums: List[int]) -> Dict:
+ params_map: Dict = {}
+ for num in cluster_nums:
+ # kylin stack
+ kylin_map = self.spark_metric_of_kylin_param_map_by_cluster_num(num)
+ if kylin_map:
+ params_map.update(kylin_map)
+ return params_map
+
+ def prometheus_spark_metric_params_of_spark_in_cluster(self, cluster_nums: List[int]) -> Dict:
+ params_map: Dict = {}
+ for num in cluster_nums:
+ # spark master
+ spark_master_map = self.metric_of_spark_master_param_map_by_num(num)
+ if spark_master_map:
+ params_map.update(spark_master_map)
+ return params_map
+
+ def kylin_param_map_by_cluster_num(self, num: int) -> Dict:
+ # kylin stack
+ kylin_stack = self.generate_stack_of_scaled_cluster_for_kylin(num)
+ kylin_ip = self.get_kylin_private_ip_by_name(kylin_stack)
+ if not kylin_ip:
+ return {}
+ kylin_map = {Params.KYLIN_NAME_IN_CLUSTER.value.format(cluster=num): kylin_ip}
+ return kylin_map
+
+ def spark_metric_of_kylin_param_map_by_cluster_num(self, num: int) -> Dict:
+ # kylin stack
+ kylin_stack = self.generate_stack_of_scaled_cluster_for_kylin(num)
+ kylin_ip = self.get_kylin_private_ip_by_name(kylin_stack)
+ if not kylin_ip:
+ return {}
+ kylin_map = {Params.KYLIN_SPARK_DIRVER_IN_CLUSTER.value.format(cluster=num): kylin_ip}
+ return kylin_map
+
+ def spark_master_param_map_by_num(self, num: int) -> Dict:
+ # spark master
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(num)
+ spark_master_ip = self.get_spark_master_host_by_name(spark_master_stack)
+ if not spark_master_ip:
+ return {}
+ spark_master_map = {Params.SPARK_MASTER_NAME_IN_CLUSTER.value.format(cluster=num): spark_master_ip}
+ return spark_master_map
+
+ def metric_of_spark_master_param_map_by_num(self, num: int) -> Dict:
+ # spark master
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(num)
+ spark_master_ip = self.get_spark_master_host_by_name(spark_master_stack)
+ if not spark_master_ip:
+ return {}
+ spark_master_map = {Params.SPARK_MASTER_METRIC_IN_CLUSTER.value.format(cluster=num): spark_master_ip}
+ return spark_master_map
+
+ def zks_param_map_by_num(self, num: int) -> Dict:
+ # zookeeper
+ zk_stack = self.generate_stack_of_scaled_cluster_for_zk(num)
+ zk_ips = self.get_instance_ips_of_zks_stack_by_name(zk_stack)
+ if not zk_ips:
+ return {}
+ zks_map = {
+ Params.ZOOKEEPER_NAME_IN_CLUSTER.value.format(num=i, cluster=num): zk_ips[i - 1] for i in range(1, 4)
+ }
+ return zks_map
+
+ def spark_worker_param_map(self, num: int) -> Dict:
+ # spark worker
+ spark_worker = self.generate_stack_of_scaled_cluster_for_spark_worker(num)
+ spark_slaves_ips = self.get_instance_ips_of_spark_slaves_stack_by_name(spark_worker)
+ if not spark_slaves_ips:
+ return {}
+ spark_slaves_map = {
+ Params.SPARK_WORKER_NAME_IN_CLUSTER.value.format(num=i, cluster=num): spark_slaves_ips[i - 1]
+ for i in range(1, 4)
+ }
+ return spark_slaves_map
+
+ def zk_param_map(self) -> Dict:
+ return dict(zip(self.get_instance_ids_of_zks_stack(), self.get_instance_ips_of_zks_stack()))
+
+ def static_services_param_map(self) -> Dict:
+ return dict(zip(self.instance_id_of_static_services, self.get_static_services_private_ip()))
+
+ def spark_master_param_map(self) -> Dict:
+ return dict(zip(self.get_spark_master_instance_id(), self.get_spark_master_host()))
+
+ def spark_slaves_param_map(self) -> Dict:
+ return dict(zip(self.get_instance_ids_of_spark_slave_stack(), self.get_instance_ips_of_spark_slaves_stack()))
+
+ def kylin_param_map(self) -> Dict:
+ return dict(zip(self.get_kylin_instance_id(), self.get_kylin_private_ip()))
+
+ def get_scaled_node_private_ips(self, cluster_num: int = None) -> [List, List]:
+ if cluster_num:
+ scaled_kylin_stacks = self.scaled_target_kylin_stacks(cluster_num)
+ scaled_spark_worker_stacks = self.scaled_target_spark_workers_stacks(cluster_num)
+ else:
+ # Always get the full scale nodes in the `UP` range
+ scaled_kylin_stacks = self.scaled_kylin_stacks
+ scaled_spark_worker_stacks = self.scaled_spark_workers_stacks
+
+ scaled_kylin_ips = []
+ for stack in scaled_kylin_stacks:
+ ip = self.get_scaled_kylin_private_ip(stack)
+ if not ip:
+ continue
+ scaled_kylin_ips.append(ip)
+
+ scaled_workers_ips = []
+ for stack in scaled_spark_worker_stacks:
+ ip = self.get_scaled_spark_worker_private_ip(stack)
+ if not ip:
+ continue
+ scaled_workers_ips.append(ip)
+ return scaled_kylin_ips, scaled_workers_ips
+
+ def get_specify_resource_from_output(self, stack_name: str, resource_type: str) -> str:
+ output = self.get_stack_output(stack_name)
+ return output[resource_type]
+
+ def get_stack_output(self, stack_name: str) -> Dict:
+ is_complete = self._stack_complete(stack_name)
+ if not is_complete:
+ raise Exception(f"{stack_name} is not complete, please check.")
+ output = self.cf_client.describe_stacks(StackName=stack_name)
+ """current output format:
+ {
+ 'Stacks' : [{
+ ...,
+ Outputs: [
+ {
+ 'OutputKey': 'xxx',
+ 'OutputValue': 'xxx',
+ 'Description': ...
+ },
+ ...]
+ }],
+ 'ResponseMEtadata': {...}
+ }
+ """
+ handled_outputs = {entry['OutputKey']: entry['OutputValue']
+ for entry in list(output['Stacks'][0]['Outputs'])}
+
+ return handled_outputs
+
+ def alive_nodes(self, cluster_nums: List = None) -> None:
+ # default cluster
+ static_msg = self.get_static_services_basic_msg()
+ kylin_msg = self.get_kylin_basic_msg()
+ spark_master_msg = self.get_spark_master_msg()
+
+ msgs = [m for m in [static_msg, kylin_msg, spark_master_msg] if m]
+
+ spark_slaves_msg = self.get_spark_slaves_basic_msg()
+ zks_msg = self.get_zks_basic_msg()
+
+ scaled_kylins_msg = self.get_scaled_kylin_basic_msg()
+ scaled_spark_workers_msg = self.get_scaled_spark_workers_basic_msg()
+
+ msgs.extend(zks_msg)
+ msgs.extend(spark_slaves_msg)
+ msgs.extend(scaled_kylins_msg)
+ msgs.extend(scaled_spark_workers_msg)
+
+ # scaled clusters nodes
+ if cluster_nums is None:
+ cluster_nums = []
+ for num in cluster_nums:
+ zks_msg_of_target_cluster = self.get_zks_of_target_cluster_msg(num)
+ msgs.extend(zks_msg_of_target_cluster)
+
+ spark_master_msg_of_target_cluster = self.get_spark_master_of_target_cluster_msg(num)
+ kylin_msg_of_target_cluster = self.get_kylin_basic_msg_of_target_cluster(num)
+
+ for msg in [spark_master_msg_of_target_cluster, kylin_msg_of_target_cluster]:
+ if not msg:
+ continue
+ msgs.append(msg)
+
+ spark_workers_msgs_of_target_cluster = self.get_spark_workers_of_target_cluster_msg(num)
+ msgs.extend(spark_workers_msgs_of_target_cluster)
+
+ scaled_kylins_msg_of_target_cluster = self.get_scaled_kylin_basic_msg_of_target_cluster(num)
+ scaled_workers_msg_of_target_cluster = self.get_scaled_spark_workers_basic_msg_of_target_cluster(num)
+ msgs.extend(scaled_kylins_msg_of_target_cluster)
+ msgs.extend(scaled_workers_msg_of_target_cluster)
+
+ header_msg = '\n=================== List Alive Nodes ===========================\n'
+ result = header_msg + f"Stack Name\t\tInstance ID\t\tPrivate Ip\t\tPublic Ip\t\t\n"
+ for msg in msgs:
+ result += msg + '\n'
+ result += header_msg
+ logger.info(result)
+
+ def is_ec2_stacks_ready(self) -> bool:
+ if not (
+ self.is_stack_complete(self.vpc_stack_name)
+ and self.is_stack_complete(self.rds_stack_name)
+ and self.is_stack_complete(self.static_service_stack_name)
+ and self.is_stack_complete(self.zk_stack_name)
+ and self.is_stack_complete(self.spark_master_stack_name)
+ and self.is_stack_complete(self.spark_slave_stack_name)
+ and self.is_stack_complete(self.kylin_stack_name)
+ ):
+ return False
+ return True
+
+ def is_target_ec2_stacks_ready(self, cluster_num: int) -> bool:
+ # Note:
+ # target cluster already must contains zookeeper stack,
+ # spark master stack, spark worker stack and kylin_stack
+ zk_stack = self.generate_stack_of_scaled_cluster_for_zk(cluster_num)
+ kylin_stack = self.generate_stack_of_scaled_cluster_for_kylin(cluster_num)
+ spark_master_stack = self.generate_stack_of_scaled_cluster_for_spark_master(cluster_num)
+ spark_worker_stack = self.generate_stack_of_scaled_cluster_for_spark_worker(cluster_num)
+ if not (
+ self.is_stack_complete(zk_stack)
+ and self.is_stack_complete(kylin_stack)
+ and self.is_stack_complete(spark_master_stack)
+ and self.is_stack_complete(spark_worker_stack)
+ ):
+ return False
+
+ return True
+
+ def is_prepared_for_scale_cluster(self) -> bool:
+ if not (
+ self.is_stack_complete(self.vpc_stack_name)
+ and self.is_stack_complete(self.rds_stack_name)
+ and self.is_stack_complete(self.static_service_stack_name)
+ ):
+ return False
+ return True
+
+ def is_ec2_stacks_terminated(self) -> bool:
+ deleted_cost_stacks: bool = (
+ self.is_stack_deleted_complete(self.static_service_stack_name)
+ and self.is_stack_deleted_complete(self.zk_stack_name)
+ and self.is_stack_deleted_complete(self.spark_master_stack_name)
+ and self.is_stack_deleted_complete(self.spark_slave_stack_name)
+ and self.is_stack_deleted_complete(self.kylin_stack_name)
+ )
+ if not deleted_cost_stacks:
+ return False
+ if not self.config['ALWAYS_DESTROY_ALL'] \
+ or self.is_stack_deleted_complete(self.vpc_stack_name):
+ return True
+ return False
+
+ def send_command(self, **kwargs) -> Dict:
+ instance_ids = kwargs['vm_name']
+ script = kwargs['script']
+ document_name = "AWS-RunShellScript"
+ parameters = {'commands': [script]}
+ response = self.ssm_client.send_command(
+ InstanceIds=instance_ids,
+ DocumentName=document_name,
+ Parameters=parameters
+ )
+ return response
+
+ def get_command_invocation(self, command_id, instance_id) -> Dict:
+ response = self.ssm_client.get_command_invocation(CommandId=command_id, InstanceId=instance_id)
+ return response
+
+ def exec_script_instance_and_return(self, name_or_id: str, script: str, timeout: int = 20) -> Dict:
+ vm_name = None
+ if isinstance(name_or_id, str):
+ vm_name = [name_or_id]
+ response = self.send_command(vm_name=vm_name, script=script)
+ command_id = response['Command']['CommandId']
+ time.sleep(5)
+ start = time.time()
+ output = None
+ while time.time() - start < timeout * 60:
+ output = self.get_command_invocation(
+ command_id=command_id,
+ instance_id=name_or_id,
+ )
+ if output['Status'] in ['Delayed', 'Success', 'Cancelled', 'TimedOut', 'Failed']:
+ break
+ time.sleep(10)
+ assert output and output['Status'] == 'Success', \
+ f"execute script failed, failed info: {output['StandardErrorContent']}"
+ return output
+
+ def stop_ec2_instance(self, instance_id: str):
+ self.ec2_client.stop_instances(
+ InstanceIds=[
+ instance_id,
+ ],
+ Force=True
+ )
+
+ def stop_ec2_instances(self, instance_ids: List):
+ self.ec2_client.stop_instances(
+ InstanceIds=instance_ids,
+ Force=True
+ )
+
+ def start_ec2_instance(self, instance_id: str) -> Dict:
+ resp = self.ec2_client.start_instances(
+ InstanceIds=[instance_id]
+ )
+ return resp
+
+ def start_ec2_instances(self, instance_ids: List) -> Dict:
+ resp = self.ec2_client.start_instances(
+ InstanceIds=instance_ids,
+ )
+ return resp
+
+ def ec2_instance_statuses(self, instance_ids: List) -> Dict:
+ resp = self.ec2_client.describe_instance_status(
+ Filters=[{
+ 'Name': 'instance-state-name',
+ 'Values': ['pending', 'running', 'shutting-down', 'terminated', 'stopping', 'stopped'],
+ }],
+ InstanceIds=instance_ids,
+ # Note: IncludeAllInstances (boolean), Default is false.
+ # When `true` , includes the health status for all instances.
+ # When `false` , includes the health status for running instances only.
+ IncludeAllInstances=True,
+ )
+ return resp['InstanceStatuses']
+
+ def ec2_instance_status(self, instance_id: str) -> str:
+ resp = self.ec2_instance_statuses(instance_ids=[instance_id])
+ assert resp, 'Instance statuses must be not empty.'
+ return resp[0]['InstanceState']['Name']
+
+ def is_ec2_instance_running(self, instance_id: str) -> bool:
+ return self.ec2_instance_status(instance_id) == 'running'
+
+ def is_ec2_instance_stopped(self, instance_id: str) -> bool:
+ return self.ec2_instance_status(instance_id) == 'stopped'
+
+ def is_rds_ready(self) -> bool:
+ describe_rds: Dict = self.get_rds_describe()
+ if not describe_rds:
+ return False
+ rds_endpoints: Dict = describe_rds['Endpoint']
+ # TODO: check rds with password and user is accessible.
+
+ is_rds_available: bool = describe_rds['DBInstanceStatus'] == 'available'
+ is_rds_matched_port = str(rds_endpoints['Port']) == self.db_port
+ return is_rds_available and is_rds_matched_port
+
+ def valid_s3_bucket(self) -> None:
+ if not self.is_s3_directory_exists(self.bucket, self.bucket_dir):
+ msg = f'Invalid S3 bucket path: {self.bucket_full_path}, please check.'
+ raise Exception(msg)
+
+ def valid_iam_role(self) -> None:
+ if not self.is_iam_role_exists():
+ msg = f'Invalid IAM role: {self.iam_role}, please check.'
+ raise Exception(msg)
+
+ def is_iam_role_exists(self) -> bool:
+ try:
+ self.iam_client.get_role(RoleName=self.iam_role)
+ return True
+ except self.iam_client.exceptions.NoSuchEntityException:
+ return False
+
+ def valid_key_pair(self) -> None:
+ if not self.is_key_pair_exists():
+ msg = f'Invalid KeyPair: {self.key_pair}. please check.'
+ raise Exception(msg)
+
+ def valid_cidr_ip(self) -> None:
+ if not self.is_valid_cidr_ip():
+ msg = f'Invalid CidrIp: {self.cidr_ip}. please check.'
+ raise Exception(msg)
+
+ def is_key_pair_exists(self) -> bool:
+ try:
+ self.ec2_client.describe_key_pairs(KeyNames=[self.key_pair])
+ return True
+ except ClientError:
+ return False
+
+ def is_valid_cidr_ip(self) -> bool:
+ if not self.cidr_ip:
+ return False
+
+ pattern = r'^([0-9]{1,3}\.){3}[0-9]{1,3}($|/(16|24|32))$'
+ res = re.match(pattern=pattern, string=self.cidr_ip)
+ if not res:
+ logger.error(f'Current {self.cidr_ip} not match correct pattern of CidrIp, please check.')
+ return False
+ return True
+
+ def is_object_exists_on_s3(self, filename: str, bucket: str, bucket_dir: str) -> bool:
+ try:
+ self.s3_client.head_object(Bucket=bucket, Key=bucket_dir + filename)
+ except botocore.exceptions.ClientError as ex:
+ assert ex.response['Error']['Code'] == '404'
+ return False
+ return True
+
+ def is_s3_directory_exists(self, bucket: str, bucket_dir: str) -> bool:
+ if bucket_dir.endswith('/'):
+ bucket_dir = bucket_dir.rstrip('/')
+
+ resp = self.s3_client.list_objects(Bucket=bucket, Prefix=bucket_dir, Delimiter='/', MaxKeys=1)
+ return 'CommonPrefixes' in resp
+
+ def upload_tars_to_s3(self, tars: List) -> None:
+ for tar in tars:
+ if self.is_object_exists_on_s3(tar, self.bucket, self.bucket_tars_dir):
+ logger.info(f'{tar} already exists, skip upload it.')
+ continue
+ self.upload_file_to_s3(TARS_PATH, tar, self.bucket, self.bucket_tars_dir)
+
+ def check_tars_on_s3(self, tars: List) -> None:
+ for tar in tars:
+ assert self.is_object_exists_on_s3(tar, self.bucket, self.bucket_tars_dir), \
+ f'{tar} not exists, please check.'
+
+ def upload_jars_to_s3(self, jars: List) -> None:
+ for jar in jars:
+ if self.is_object_exists_on_s3(jar, self.bucket, self.bucket_jars_dir):
+ logger.info(f'{jar} already exists, skip upload it.')
+ continue
+ self.upload_file_to_s3(JARS_PATH, jar, self.bucket, self.bucket_jars_dir)
+
+ def check_jars_on_s3(self, jars: List) -> None:
+ for jar in jars:
+ assert self.is_object_exists_on_s3(jar, self.bucket, self.bucket_jars_dir), \
+ f'{jar} not exists, please check.'
+
+ def upload_scripts_to_s3(self, scripts: List) -> None:
+ for script in scripts:
+ if self.is_object_exists_on_s3(script, self.bucket, self.bucket_scripts_dir):
+ logger.info(f'{script} already exists, skip upload it.')
+ continue
+ self.upload_file_to_s3(SCRIPTS_PATH, script, self.bucket, self.bucket_scripts_dir)
+
+ def upload_kylin_properties_to_s3(self, cluster_num: int = None, properties_file: str = 'kylin.properties') -> None:
+ # Always to upload local kylin.properties
+ local_dir = KYLIN_PROPERTIES_TEMPLATE_DIR.format(cluster_num=cluster_num if cluster_num else 'default')
+ bucket_dir = self.bucket_kylin_properties_of_other_cluster_dir.format(
+ cluster_num=cluster_num if cluster_num else 'default')
+ self.upload_file_to_s3(
+ local_file_dir=local_dir,
+ filename=properties_file,
+ bucket=self.bucket,
+ bucket_dir=bucket_dir)
+
+ def check_scripts_on_s3(self, scripts: List) -> None:
+ for script in scripts:
+ assert self.is_object_exists_on_s3(script, self.bucket, self.bucket_scripts_dir), \
+ f'{script} not exists, please check.'
+
+ def upload_file_to_s3(self, local_file_dir: str, filename: str, bucket: str, bucket_dir: str) -> None:
+ logger.info(f'Uploading {filename} from {local_file_dir} to S3 bucket: {bucket}/{bucket_dir}.')
+ self.s3_client.upload_file(os.path.join(local_file_dir, filename), bucket, bucket_dir + filename)
+ logger.info(f'Uploaded {filename} successfully.')
+
+ def is_stack_deleted_complete(self, stack_name: str) -> bool:
+ if self.is_stack_create_complete(stack_name):
+ # return directly if current stack status is 'CREATE_COMPLETE'
+ return False
+ if self._stack_delete_complete(stack_name):
+ return True
+ return False
+
+ def is_stack_create_complete(self, stack_name: str) -> bool:
+ return self._stack_status_check(name_or_id=stack_name, status='CREATE_COMPLETE')
+
+ def is_stack_delete_complete(self, stack_name: str) -> bool:
+ return self._stack_status_check(name_or_id=stack_name, status='DELETE_COMPLETE')
+
+ def is_stack_complete(self, stack_name: str) -> bool:
+ if self._stack_complete(stack_name):
+ return True
+ return False
+
+ def _validate_spark_worker_scale(self, stack_name: str) -> None:
+ if stack_name not in self.scaled_spark_workers_stacks:
+ msg = f'{stack_name} not in scaled list, please check.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ def _validate_spark_worker_of_target_cluster_scale(self, stack_name: str, cluster_num: int) -> None:
+ if stack_name not in self.scaled_target_spark_workers_stacks(cluster_num):
+ msg = f'{stack_name} not in scaled list of target cluster {cluster_num}, please check.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ def _validate_kylin_scale(self, stack_name: str) -> None:
+ if stack_name not in self.scaled_kylin_stacks:
+ msg = f'{stack_name} not in scaled list, please check.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ def _validate_kylin_of_target_cluster_scale(self, stack_name: str, cluster_num: int) -> None:
+ if stack_name not in self.scaled_target_kylin_stacks(cluster_num):
+ msg = f'{stack_name} not in scaled list of target cluster {cluster_num}, please check.'
+ logger.error(msg)
+ raise Exception(msg)
+
+ def _stack_status_check(self, name_or_id: str, status: str) -> bool:
+ try:
+ resp: Dict = self.cf_client.describe_stacks(StackName=name_or_id)
+ except ClientError:
+ return False
+ return resp['Stacks'][0]['StackStatus'] == status
+
+ def _stack_complete(self, stack_name: str) -> bool:
+ try:
+ self.create_complete_waiter.wait(
+ StackName=stack_name,
+ WaiterConfig={
+ 'Delay': 30,
+ 'MaxAttempts': 120
+ }
+ )
+ except WaiterError as wx:
+ # logger.error(wx)
+ return False
+ return True
... 717 lines suppressed ...