You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/09/05 08:35:01 UTC
git commit: [SPARK-3391][EC2] Support attaching up to 8 EBS volumes.
Repository: spark
Updated Branches:
refs/heads/master 1904bac38 -> 1725a1a5d
[SPARK-3391][EC2] Support attaching up to 8 EBS volumes.
Please merge this at the same time as https://github.com/mesos/spark-ec2/pull/66
Author: Reynold Xin <rx...@apache.org>
Closes #2260 from rxin/ec2-ebs-vol and squashes the following commits:
b9527d9 [Reynold Xin] Removed io1 ebs type.
bf9c403 [Reynold Xin] Made EBS volume type configurable.
c8e25ea [Reynold Xin] Support up to 8 EBS volumes.
adf4f2e [Reynold Xin] Revert git repo change.
020c542 [Reynold Xin] [SPARK-3391] Support attaching more than 1 EBS volumes.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1725a1a5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1725a1a5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1725a1a5
Branch: refs/heads/master
Commit: 1725a1a5d10a53762bd80f391eddbf306f2841ee
Parents: 1904bac
Author: Reynold Xin <rx...@apache.org>
Authored: Thu Sep 4 23:34:58 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Thu Sep 4 23:34:58 2014 -0700
----------------------------------------------------------------------
ec2/spark_ec2.py | 33 +++++++++++++++++++++++++--------
1 file changed, 25 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1725a1a5/ec2/spark_ec2.py
----------------------------------------------------------------------
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index eed6eb8..1670fac 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -102,9 +102,17 @@ def parse_args():
"(for debugging)")
parser.add_option(
"--ebs-vol-size", metavar="SIZE", type="int", default=0,
- help="Attach a new EBS volume of size SIZE (in GB) to each node as " +
- "/vol. The volumes will be deleted when the instances terminate. " +
- "Only possible on EBS-backed AMIs.")
+ help="Size (in GB) of each EBS volume.")
+ parser.add_option(
+ "--ebs-vol-type", default="standard",
+ help="EBS volume type (e.g. 'gp2', 'standard').")
+ parser.add_option(
+ "--ebs-vol-num", type="int", default=1,
+ help="Number of EBS volumes to attach to each node as /vol[x]. " +
+ "The volumes will be deleted when the instances terminate. " +
+ "Only possible on EBS-backed AMIs. " +
+ "EBS volumes are only attached if --ebs-vol-size > 0." +
+ "Only support up to 8 EBS volumes.")
parser.add_option(
"--swap", metavar="SWAP", type="int", default=1024,
help="Swap space to set up per node, in MB (default: 1024)")
@@ -348,13 +356,16 @@ def launch_cluster(conn, opts, cluster_name):
print >> stderr, "Could not find AMI " + opts.ami
sys.exit(1)
- # Create block device mapping so that we can add an EBS volume if asked to
+ # Create block device mapping so that we can add EBS volumes if asked to.
+ # The first drive is attached as /dev/sds, 2nd as /dev/sdt, ... /dev/sdz
block_map = BlockDeviceMapping()
if opts.ebs_vol_size > 0:
- device = EBSBlockDeviceType()
- device.size = opts.ebs_vol_size
- device.delete_on_termination = True
- block_map["/dev/sdv"] = device
+ for i in range(opts.ebs_vol_num):
+ device = EBSBlockDeviceType()
+ device.size = opts.ebs_vol_size
+ device.volume_type=opts.ebs_vol_type
+ device.delete_on_termination = True
+ block_map["/dev/sd" + chr(ord('s') + i)] = device
# AWS ignores the AMI-specified block device mapping for M3 (see SPARK-3342).
if opts.instance_type.startswith('m3.'):
@@ -828,6 +839,12 @@ def get_partition(total, num_partitions, current_partitions):
def real_main():
(opts, action, cluster_name) = parse_args()
+
+ # Input parameter validation
+ if opts.ebs_vol_num > 8:
+ print >> stderr, "ebs-vol-num cannot be greater than 8"
+ sys.exit(1)
+
try:
conn = ec2.connect_to_region(opts.region)
except Exception as e:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org