You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Vladimir Grigor (JIRA)" <ji...@apache.org> on 2015/04/04 06:46:33 UTC
[jira] [Updated] (SPARK-925) Allow ec2 scripts to load default
options from a json file
[ https://issues.apache.org/jira/browse/SPARK-925?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Vladimir Grigor updated SPARK-925:
----------------------------------
Description:
The option list for ec2 script can be a little irritating to type in, especially things like path to identity-file, region , zone, ami etc.
It would be nice if ec2 script looks for an options.json file in the following order: (1) CWD, (2) ~/spark-ec2, (3) same dir as spark_ec2.py
Something like:
def get_defaults_from_options():
# Check to see if a options.json file exists, if so load it.
# However, values in the options.json file can only overide values in opts
# if the Opt values are None or ""
# i.e. commandline options take presidence
defaults = {'aws-access-key-id':'','aws-secret-access-key':'','key-pair':'', 'identity-file':'', 'region':'ap-southeast-1', 'zone':'', 'ami':'','slaves':1, 'instance-type':'m1.large'}
# Look for options.json in directory cluster was called from
# Had to modify the spark_ec2 wrapper script since it mangles the pwd
startwd = os.environ['STARTWD']
if os.path.exists(os.path.join(startwd,"options.json")):
optionspath = os.path.join(startwd,"options.json")
else:
optionspath = os.path.join(os.getcwd(),"options.json")
try:
print "Loading options file: ", optionspath
with open (optionspath) as json_data:
jdata = json.load(json_data)
for k in jdata:
defaults[k]=jdata[k]
except IOError:
print 'Warning: options.json file not loaded'
# Check permissions on identity-file, if defined, otherwise launch will fail late and will be irritating
if defaults['identity-file']!='':
st = os.stat(defaults['identity-file'])
user_can_read = bool(st.st_mode & stat.S_IRUSR)
grp_perms = bool(st.st_mode & stat.S_IRWXG)
others_perm = bool(st.st_mode & stat.S_IRWXO)
if (not user_can_read):
print "No read permission to read ", defaults['identify-file']
sys.exit(1)
if (grp_perms or others_perm):
print "Permissions are too open, please chmod 600 file ", defaults['identify-file']
sys.exit(1)
# if defaults contain AWS access id or private key, set it to environment.
# required for use with boto to access the AWS console
if defaults['aws-access-key-id'] != '':
os.environ['AWS_ACCESS_KEY_ID']=defaults['aws-access-key-id']
if defaults['aws-secret-access-key'] != '':
os.environ['AWS_SECRET_ACCESS_KEY'] = defaults['aws-secret-access-key']
return defaults
was:
The option list for ec2 script can be a little irritating to type in, especially things like path to identity-file, region , zone, ami etc.
It would be nice if ec2 script looks for an options.json file in the following order: (1) PWD, (2) ~/spark-ec2, (3) same dir as spark_ec2.py
Something like:
def get_defaults_from_options():
# Check to see if a options.json file exists, if so load it.
# However, values in the options.json file can only overide values in opts
# if the Opt values are None or ""
# i.e. commandline options take presidence
defaults = {'aws-access-key-id':'','aws-secret-access-key':'','key-pair':'', 'identity-file':'', 'region':'ap-southeast-1', 'zone':'', 'ami':'','slaves':1, 'instance-type':'m1.large'}
# Look for options.json in directory cluster was called from
# Had to modify the spark_ec2 wrapper script since it mangles the pwd
startwd = os.environ['STARTWD']
if os.path.exists(os.path.join(startwd,"options.json")):
optionspath = os.path.join(startwd,"options.json")
else:
optionspath = os.path.join(os.getcwd(),"options.json")
try:
print "Loading options file: ", optionspath
with open (optionspath) as json_data:
jdata = json.load(json_data)
for k in jdata:
defaults[k]=jdata[k]
except IOError:
print 'Warning: options.json file not loaded'
# Check permissions on identity-file, if defined, otherwise launch will fail late and will be irritating
if defaults['identity-file']!='':
st = os.stat(defaults['identity-file'])
user_can_read = bool(st.st_mode & stat.S_IRUSR)
grp_perms = bool(st.st_mode & stat.S_IRWXG)
others_perm = bool(st.st_mode & stat.S_IRWXO)
if (not user_can_read):
print "No read permission to read ", defaults['identify-file']
sys.exit(1)
if (grp_perms or others_perm):
print "Permissions are too open, please chmod 600 file ", defaults['identify-file']
sys.exit(1)
# if defaults contain AWS access id or private key, set it to environment.
# required for use with boto to access the AWS console
if defaults['aws-access-key-id'] != '':
os.environ['AWS_ACCESS_KEY_ID']=defaults['aws-access-key-id']
if defaults['aws-secret-access-key'] != '':
os.environ['AWS_SECRET_ACCESS_KEY'] = defaults['aws-secret-access-key']
return defaults
> Allow ec2 scripts to load default options from a json file
> ----------------------------------------------------------
>
> Key: SPARK-925
> URL: https://issues.apache.org/jira/browse/SPARK-925
> Project: Spark
> Issue Type: Improvement
> Components: EC2
> Affects Versions: 0.8.0
> Reporter: Shay Seng
> Priority: Minor
>
> The option list for ec2 script can be a little irritating to type in, especially things like path to identity-file, region , zone, ami etc.
> It would be nice if ec2 script looks for an options.json file in the following order: (1) CWD, (2) ~/spark-ec2, (3) same dir as spark_ec2.py
> Something like:
> def get_defaults_from_options():
> # Check to see if a options.json file exists, if so load it.
> # However, values in the options.json file can only overide values in opts
> # if the Opt values are None or ""
> # i.e. commandline options take presidence
> defaults = {'aws-access-key-id':'','aws-secret-access-key':'','key-pair':'', 'identity-file':'', 'region':'ap-southeast-1', 'zone':'', 'ami':'','slaves':1, 'instance-type':'m1.large'}
> # Look for options.json in directory cluster was called from
> # Had to modify the spark_ec2 wrapper script since it mangles the pwd
> startwd = os.environ['STARTWD']
> if os.path.exists(os.path.join(startwd,"options.json")):
> optionspath = os.path.join(startwd,"options.json")
> else:
> optionspath = os.path.join(os.getcwd(),"options.json")
>
> try:
> print "Loading options file: ", optionspath
> with open (optionspath) as json_data:
> jdata = json.load(json_data)
> for k in jdata:
> defaults[k]=jdata[k]
> except IOError:
> print 'Warning: options.json file not loaded'
> # Check permissions on identity-file, if defined, otherwise launch will fail late and will be irritating
> if defaults['identity-file']!='':
> st = os.stat(defaults['identity-file'])
> user_can_read = bool(st.st_mode & stat.S_IRUSR)
> grp_perms = bool(st.st_mode & stat.S_IRWXG)
> others_perm = bool(st.st_mode & stat.S_IRWXO)
> if (not user_can_read):
> print "No read permission to read ", defaults['identify-file']
> sys.exit(1)
> if (grp_perms or others_perm):
> print "Permissions are too open, please chmod 600 file ", defaults['identify-file']
> sys.exit(1)
> # if defaults contain AWS access id or private key, set it to environment.
> # required for use with boto to access the AWS console
> if defaults['aws-access-key-id'] != '':
> os.environ['AWS_ACCESS_KEY_ID']=defaults['aws-access-key-id']
> if defaults['aws-secret-access-key'] != '':
> os.environ['AWS_SECRET_ACCESS_KEY'] = defaults['aws-secret-access-key']
> return defaults
>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org