You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ya...@apache.org on 2022/04/12 08:48:25 UTC
[kylin] branch kylin4_on_cloud updated: # minor fix log (#1856)
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a commit to branch kylin4_on_cloud
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin4_on_cloud by this push:
new e630a46dfb # minor fix log (#1856)
e630a46dfb is described below
commit e630a46dfbbe2170774a87e811c5840bfc10f083
Author: Tengting Xu <34...@users.noreply.github.com>
AuthorDate: Tue Apr 12 16:48:20 2022 +0800
# minor fix log (#1856)
* # minor fix log
* # minor fix, enhance for checking stack status
* # minor fix log
* # minor fix, enhance for checking stack status
* # minor fix file path check
* # minor fix, correct spark package for kylin
---
client/basic.py | 2 +-
instances/aws_instance.py | 40 +++++++++++++++++++++-------------------
instances/kylin_utils.py | 2 +-
utils/common_utils.py | 6 ++++--
utils/engine_utils.py | 5 +++--
5 files changed, 30 insertions(+), 25 deletions(-)
diff --git a/client/basic.py b/client/basic.py
index 84bfdcb6ca..53058f8a55 100644
--- a/client/basic.py
+++ b/client/basic.py
@@ -121,7 +121,7 @@ class BasicHttpClient:
return data.get('data', data)
return resp.text
except requests.exceptions.ReadTimeout as timeout_error:
- logger.error(timeout_error)
+ logger.warning(timeout_error)
time.sleep(60)
except requests.HTTPError as http_error:
err_msg = f"{str(http_error)} [return code: {data.get('code', '')}]-[{data.get('msg', '')}]\n" \
diff --git a/instances/aws_instance.py b/instances/aws_instance.py
index 32c4057997..a43a025d70 100644
--- a/instances/aws_instance.py
+++ b/instances/aws_instance.py
@@ -376,7 +376,7 @@ class AWSInstance:
# ============ VPC Services Start ============
def create_vpc_stack(self) -> Optional[Dict]:
- if self.is_stack_complete(self.vpc_stack_name):
+ if self.is_stack_exists(self.vpc_stack_name) and self.is_stack_complete(self.vpc_stack_name):
return
params: Dict = self.config[Config.EC2_VPC_PARAMS.value]
params[Params.CIDR_IP.value] = self.cidr_ip
@@ -412,9 +412,9 @@ class AWSInstance:
return self.is_db_available(self.db_identifier)
def create_rds_stack(self) -> Optional[Dict]:
- if self.is_stack_complete(self.rds_stack_name):
- return
- if self.is_rds_exists():
+ if self.is_stack_exists(self.rds_stack_name) \
+ and self.is_stack_complete(self.rds_stack_name)\
+ and self.is_rds_exists():
logger.warning(f'db {self.db_identifier} already exists.')
return
params: Dict = self.config[Config.EC2_RDS_PARAMS.value]
@@ -445,7 +445,7 @@ class AWSInstance:
logger.warning(msg)
raise Exception(msg)
- if self.is_stack_complete(self.static_service_stack_name):
+ if self.is_stack_exists(self.static_service_stack_name) and self.is_stack_complete(self.static_service_stack_name):
return
params: Dict = self.config[Config.EC2_STATIC_SERVICES_PARAMS.value]
# update needed params
@@ -529,7 +529,7 @@ class AWSInstance:
else:
zk_stack_name = self.zk_stack_name
- if self.is_stack_complete(zk_stack_name):
+ if self.is_stack_exists(zk_stack_name) and self.is_stack_complete(zk_stack_name):
return
params: Dict = self.config[Config.EC2_ZOOKEEPERS_PARAMS.value]
# update needed params
@@ -673,7 +673,7 @@ class AWSInstance:
zk_stack = self.zk_stack_name
spark_master_stack = self.spark_master_stack_name
- if self.is_stack_complete(kylin_stack_name):
+ if self.is_stack_exists(kylin_stack_name) and self.is_stack_complete(kylin_stack_name):
return
params: Dict = self.config[Config.EC2_KYLIN4_PARAMS.value]
@@ -897,7 +897,7 @@ class AWSInstance:
else:
spark_master_stack_name = self.spark_master_stack_name
- if self.is_stack_complete(spark_master_stack_name):
+ if self.is_stack_exists(spark_master_stack_name) and self.is_stack_complete(spark_master_stack_name):
return
params: Dict = self.config[Config.EC2_SPARK_MASTER_PARAMS.value]
@@ -1967,7 +1967,7 @@ class AWSInstance:
break
time.sleep(10)
if not output or output['Status'] != 'Success':
- logger.error(output)
+ logger.warning(output)
assert output and output['Status'] == 'Success', \
f"execute script failed, failed details message: {output}"
@@ -2050,7 +2050,7 @@ class AWSInstance:
self.iam_client.get_role(RoleName=self.iam_role)
return True
except self.iam_client.exceptions.NoSuchEntityException as err:
- logger.error(f"check iam role error: {err}")
+ logger.warning(f"check iam role error: {err}")
return False
def valid_key_pair(self) -> None:
@@ -2068,7 +2068,7 @@ class AWSInstance:
self.ec2_client.describe_key_pairs(KeyNames=[self.key_pair])
return True
except ClientError as ce:
- logger.error(f"check key pair error: {ce}")
+ logger.warning(f"check key pair error: {ce}")
return False
def is_valid_cidr_ip(self) -> bool:
@@ -2087,11 +2087,11 @@ class AWSInstance:
response = self.s3_client.head_object(Bucket=bucket, Key=bucket_dir + filename)
Utils.is_uploaded_success(filename=filename, size_in_bytes=response['ContentLength'])
except botocore.exceptions.ClientError as ex:
- logger.error(f"check object exists on s3 error:{ex}")
+ logger.info(f"check object exists on s3 : File {filename} does not exist and will be uploaded locally.")
assert ex.response['Error']['Code'] == '404'
return False
- except AssertionError as ex:
- logger.error(f"check object exists on s3 error:{ex}")
+ except AssertionError:
+ logger.info(f"check object exists on s3 : File {filename} does not exist and will be uploaded locally.")
return False
return True
@@ -2185,6 +2185,11 @@ class AWSInstance:
def is_stack_rollback_in_progress(self, stack_name: str) -> bool:
return self._stack_status_check(name_or_id=stack_name, status='ROLLBACK_IN_PROGRESS')
+ def is_stack_exists(self, stack_name: str) -> bool:
+ if self._stack_exists(stack_name=stack_name):
+ return True
+ return False
+
def is_stack_complete(self, stack_name: str) -> bool:
if self._stack_complete(stack_name):
return True
@@ -2256,7 +2261,6 @@ class AWSInstance:
}
)
except WaiterError as wx:
- # logger.error(wx)
return False
return True
@@ -2265,12 +2269,11 @@ class AWSInstance:
self.exists_waiter.wait(
StackName=stack_name,
WaiterConfig={
- 'Delay': 5,
- 'MaxAttempts': 2
+ 'Delay': 10,
+ 'MaxAttempts': 3
}
)
except WaiterError:
- # logger.error(wx)
return False
return True
@@ -2284,7 +2287,6 @@ class AWSInstance:
}
)
except WaiterError as wx:
- # logger.error(wx)
return False
return True
# ============ Utils Services End ============
diff --git a/instances/kylin_utils.py b/instances/kylin_utils.py
index ccf4eb3fc7..6c6ecd6e96 100644
--- a/instances/kylin_utils.py
+++ b/instances/kylin_utils.py
@@ -34,6 +34,6 @@ class KylinUtils:
check_times=3
)
except AssertionError:
- logger.error('Check kylin status failed, please check the public ip whether is opened.')
+ logger.warning('Check kylin status failed, please check the public ip whether is opened.')
return False
return True
diff --git a/utils/common_utils.py b/utils/common_utils.py
index 14e31f4810..a784eafedc 100644
--- a/utils/common_utils.py
+++ b/utils/common_utils.py
@@ -115,8 +115,10 @@ class Utils:
r = requests.get(url, stream=True)
if not r.ok:
# HTTP status code 4XX/5XX
+ msg = "Download failed: status code {}\n{}".format(r.status_code, r.text)
logger.error("Download failed: status code {}\n{}".format(r.status_code, r.text))
- return
+ raise Exception(msg)
+
logger.info(f"Downloading {os.path.abspath(file_path)}.")
with open(file_path, 'wb') as f:
# set downloading bar
@@ -214,7 +216,7 @@ class Utils:
# delete useless kylin.properties
kylin_properties_paths = os.listdir(KYLIN_PROPERTIES_DIR)
for path in kylin_properties_paths:
- if path in ['default', 'templates']:
+ if path in ['default', 'templates', 'mode_templates']:
continue
if not cluster_nums and (cluster_nums and path not in cluster_nums):
diff --git a/utils/engine_utils.py b/utils/engine_utils.py
index 4f6ff52c5b..70ff926279 100644
--- a/utils/engine_utils.py
+++ b/utils/engine_utils.py
@@ -20,7 +20,8 @@ from typing import List
from clouds.aws import AWS
from constant.config import Config
-from constant.deployment import NodeType, ScaleType
+from constant.deployment import NodeType, ScaleType, MODE
+from constant.server_mode import ServerMode
from constant.yaml_files import Tar
from instances.kylin_utils import KylinUtils
from utils.common_utils import Utils
@@ -47,7 +48,7 @@ class EngineUtils:
hadoop_package = Tar.HADOOP.value.format(HADOOP_VERSION=self.config['HADOOP_VERSION'])
node_exporter_package = Tar.NODE.value.format(NODE_EXPORTER_VERSION=self.config['NODE_EXPORTER_VERSION'])
prometheus_package = Tar.PROMETHEUS.value.format(PROMETHEUS_VERSION=self.config['PROMETHEUS_VERSION'])
- if self.config['SUPPORT_GLUE'] == 'true':
+ if self.config['SUPPORT_GLUE'] == 'true' and self.config[MODE.KYLIN.value] == ServerMode.JOB.value:
spark_package = Tar.SPARK_FOR_GLUE.value.format(
SPARK_VERSION=self.config['SPARK_VERSION'],
HADOOP_VERSION=self.config['HADOOP_VERSION'])