You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@fluo.apache.org by GitBox <gi...@apache.org> on 2018/07/13 16:21:30 UTC
[GitHub] mikewalch closed pull request #223: Several updates to Muchos
mikewalch closed pull request #223: Several updates to Muchos
URL: https://github.com/apache/fluo-muchos/pull/223
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index c23439a..b729350 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -49,8 +49,13 @@ fluo_yarn_sha256 = c6220d35cf23127272f3b5638c44586504dc17a46f5beecdfee5027b5ff87
accumulo_sha256 = f9cebff3ff85cacb8c80263725663b047ef239916cb4490c93c62509d62e1e76
[ec2]
+# AWS machine image to use. The default below is for a CentOS 7 image (in us-east-1).
+# You may need to change this value if a new image has been released or you are running in a different region.
+# Before using this AMI, subscribe to it on the CentOS product page below or launching will fail:
+# https://aws.amazon.com/marketplace/pp/B00O7WM7QW
+aws_ami = ami-9887c6e7
# Type of AWS instance launched by default
-default_instance_type = m3.large
+default_instance_type = m5d.large
# Type of AWS instance launched for any node running 'worker' service
# Leave default below to use same instance type set by 'default_instance_type' property
worker_instance_type = %(default_instance_type)s
@@ -71,9 +76,6 @@ fstype = ext3
force_format = no
# Tags to add instances
#instance_tags = key1:value1,key2:value2
-# Overrides the default CentOS 7 AWS AMIs used by Muchos
-# Only set if you want to use a custom AMI (which should be derived from a CentOS 7 AMI)
-#aws_ami = ami-xxxxxxxx
# Nodes will be given public IP addresses if true
associate_public_ip = true
# Path to file containing user data that will be executed at launch
diff --git a/lib/muchos/config.py b/lib/muchos/config.py
index 96b5b6f..7196aa0 100644
--- a/lib/muchos/config.py
+++ b/lib/muchos/config.py
@@ -14,7 +14,7 @@
from ConfigParser import ConfigParser
from sys import exit
-from util import get_ephemeral_devices, get_arch, get_ami
+from util import get_ephemeral_devices, get_arch
import os
SERVICES = ['zookeeper', 'namenode', 'resourcemanager', 'accumulomaster', 'mesosmaster', 'worker', 'fluo', 'fluo_yarn', 'metrics']
@@ -52,9 +52,9 @@ def verify_config(self, action):
if not self.has_service(service):
exit("ERROR - Missing '{0}' service from [nodes] section of muchos.props".format(service))
- def verify_launch(self, region):
- self.get_image_id(self.get('ec2', 'default_instance_type'), region)
- self.get_image_id(self.get('ec2', 'worker_instance_type'), region)
+ def verify_launch(self):
+ self.verify_instance_type(self.get('ec2', 'default_instance_type'))
+ self.verify_instance_type(self.get('ec2', 'worker_instance_type'))
def init_nodes(self):
self.node_d = {}
@@ -121,11 +121,10 @@ def version(self, software_id):
def sha256(self, software_id):
return self.get('general', software_id + '_sha256')
- def get_image_id(self, instance_type, region):
+ def verify_instance_type(self, instance_type):
if get_arch(instance_type) == 'pvm':
exit("ERROR - Configuration contains instance type '{0}' that uses pvm architecture."
"Only hvm architecture is supported!".format(instance_type))
- return get_ami(region)
def instance_tags(self):
retd = {}
diff --git a/lib/muchos/main.py b/lib/muchos/main.py
index 97d205a..26c7025 100644
--- a/lib/muchos/main.py
+++ b/lib/muchos/main.py
@@ -56,15 +56,13 @@ def launch_node(self, hostname, services, sg_id):
request['InstanceType'] = instance_type
request['InstanceInitiatedShutdownBehavior'] = self.config.get('ec2', 'shutdown_behavior')
- if self.config.has_option('ec2', 'aws_ami'):
- image_id = self.config.get('ec2', 'aws_ami')
- else:
- session = boto3.session.Session()
- image_id = self.config.get_image_id(instance_type, session.region_name)
+ if not self.config.has_option('ec2', 'aws_ami'):
+ exit('aws_ami property must be set!')
+ image_id = self.config.get('ec2', 'aws_ami')
if not image_id:
- exit('ERROR - Image not found for instance type: '+instance_type)
- request['ImageId'] = image_id
+ exit('aws_ami property was not properly')
+ request['ImageId'] = image_id
request['BlockDeviceMappings'] = get_block_device_map(instance_type)
if self.config.has_option('ec2', 'key_name'):
@@ -155,8 +153,7 @@ def launch(self):
exit("ERROR - A hosts file already exists at {0}. Please delete before running launch again"
.format(self.config.hosts_path))
- session = boto3.session.Session()
- self.config.verify_launch(session.region_name)
+ self.config.verify_launch()
print "Launching {0} cluster".format(self.config.cluster_name)
diff --git a/lib/muchos/util.py b/lib/muchos/util.py
index 86811cb..1ddea45 100644
--- a/lib/muchos/util.py
+++ b/lib/muchos/util.py
@@ -33,15 +33,10 @@ def __init__(self, arch, ephemeral=1, has_nvme=False):
are no longer available to new users. If you think this is the case, go to the CentOS 7 product
page on AWS Marketplace at the URL below to find the latest AMI:
-https://aws.amazon.com/marketplace/ordering?productId=b7ee8a69-ee97-4a49-9e68-afaee216db2e
+ https://aws.amazon.com/marketplace/pp/B00O7WM7QW
-On the product page, click 'Manual Launch' to find the latest AMI ID for your EC2 region.
-This should be used to set the 'aws_ami' property in your muchos.props which will override
-the default AMI IDs used by Muchos. After setting the 'aws_ami' property, run the launch
-command again.
-
-Also, let us know that this has occured by creating an issue on the Muchos's GitHub page
-and we'll upgrade the defaults AMIs used by Muchos to be the latest CentOS images.
+On the product page, find the latest AMI ID for your EC2 region. This should be used to set the 'aws_ami'
+property in your muchos.props. After setting the 'aws_ami' property, run the launch command again.
"""
instance_types = {
@@ -76,6 +71,12 @@ def __init__(self, arch, ephemeral=1, has_nvme=False):
"m3.large": EC2Type("hvm"),
"m3.medium": EC2Type("hvm"),
"m3.xlarge": EC2Type("hvm", 2),
+ "m5d.large": EC2Type("hvm", 1, True),
+ "m5d.xlarge": EC2Type("hvm", 1, True),
+ "m5d.2xlarge": EC2Type("hvm", 1, True),
+ "m5d.4xlarge": EC2Type("hvm", 2, True),
+ "m5d.12xlarge": EC2Type("hvm", 2, True),
+ "m5d.24xlarge": EC2Type("hvm", 4, True),
"r3.2xlarge": EC2Type("hvm", 1),
"r3.4xlarge": EC2Type("hvm", 1),
"r3.8xlarge": EC2Type("hvm", 2),
@@ -87,24 +88,6 @@ def __init__(self, arch, ephemeral=1, has_nvme=False):
"d2.8xlarge": EC2Type("hvm", 24)
}
-# AMI given region for HVM arch. PVM arch is not supported.
-ami_lookup = {
- "us-east-1": "ami-4bf3d731",
- "us-east-2": "ami-e1496384",
- "us-west-1": "ami-65e0e305",
- "us-west-2": "ami-a042f4d8",
- "ca-central-1": "ami-dcad28b8",
- "eu-central-1": "ami-337be65c",
- "eu-west-1": "ami-6e28b517",
- "eu-west-2": "ami-ee6a718a",
- "eu-west-3": "ami-bfff49c2",
- "ap-northeast-1": "ami-25bd2743",
- "ap-northeast-2": "ami-7248e81c",
- "ap-southeast-1": "ami-d2fa88ae",
- "ap-southeast-2": "ami-b6bb47d4",
- "ap-south-1": "ami-5d99ce32",
- "sa-east-1": "ami-f9adef95"
-}
def verify_type(instance_type):
if instance_type not in instance_types:
@@ -125,7 +108,11 @@ def get_ephemeral_devices(instance_type):
devices = []
ec2_type = instance_types.get(instance_type)
- for i in range(0, ec2_type.ephemeral):
+ start = 0
+ if instance_type.startswith("m5d"):
+ start = 1
+
+ for i in range(start, ec2_type.ephemeral + start):
if ec2_type.has_nvme:
devices.append('/dev/nvme' + str(i) + 'n1')
else:
@@ -148,9 +135,6 @@ def get_block_device_map(instance_type):
return bdm
-def get_ami(region):
- return ami_lookup.get(region)
-
def parse_args(hosts_dir, input_args=None):
parser = OptionParser(
usage="muchos [options] <action>\n\n"
diff --git a/lib/tests/test_config.py b/lib/tests/test_config.py
index 717048e..6ef2f32 100644
--- a/lib/tests/test_config.py
+++ b/lib/tests/test_config.py
@@ -18,12 +18,13 @@
def test_defaults():
c = DeployConfig("muchos", '../conf/muchos.props.example', '../conf/hosts/example/example_cluster',
'mycluster')
- assert c.get('ec2', 'default_instance_type') == 'm3.large'
- assert c.get('ec2', 'worker_instance_type') == 'm3.large'
+ assert c.get('ec2', 'default_instance_type') == 'm5d.large'
+ assert c.get('ec2', 'worker_instance_type') == 'm5d.large'
+ assert c.get('ec2', 'aws_ami') == 'ami-9887c6e7'
assert c.max_ephemeral() == 1
assert c.mounts(2) == ['/media/ephemeral0', '/media/ephemeral1']
- assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0', ], 'devices': ['/dev/xvdb', ]},
- 'worker': {'mounts': ['/media/ephemeral0', ], 'devices': ['/dev/xvdb', ]}}
+ assert c.node_type_map() == {'default': {'mounts': ['/media/ephemeral0', ], 'devices': ['/dev/nvme1n1', ]},
+ 'worker': {'mounts': ['/media/ephemeral0', ], 'devices': ['/dev/nvme1n1', ]}}
assert c.node_type('worker1') == 'worker'
assert c.node_type('leader1') == 'default'
assert not c.has_option('ec2', 'vpc_id')
@@ -59,7 +60,6 @@ def test_defaults():
assert c.get_host_services() == [('leader1', 'namenode zookeeper fluo fluo_yarn'), ('leader2', 'resourcemanager zookeeper'),
('leader3', 'accumulomaster zookeeper'), ('metrics', 'metrics'),
('worker1', 'worker'), ('worker2', 'worker'), ('worker3', 'worker')]
- assert c.get_image_id('m3.large', 'us-east-1') == 'ami-4bf3d731'
def test_case_sensitive():
diff --git a/lib/tests/test_util.py b/lib/tests/test_util.py
index 6c6015d..9fc3c7c 100644
--- a/lib/tests/test_util.py
+++ b/lib/tests/test_util.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from muchos.util import get_arch, parse_args, get_ami, get_ephemeral_devices
+from muchos.util import get_arch, parse_args, get_ephemeral_devices
def test_util():
@@ -25,8 +25,6 @@ def test_util():
assert get_arch('m1.large') == 'pvm'
assert get_arch('m3.large') == 'hvm'
- assert get_ami('us-east-1') == 'ami-4bf3d731'
-
hosts_dir = '../conf/hosts'
assert parse_args(hosts_dir, ['launch']) is None
assert parse_args(hosts_dir, ['launch', 'mycluster']) is None
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services