You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sv...@apache.org on 2017/08/02 16:39:30 UTC
[1/4] incubator-trafodion git commit: [TRAFODION-2697] Set bulkload
permissions for hbase upload
Repository: incubator-trafodion
Updated Branches:
refs/heads/master ed8666d18 -> d86a0baf3
[TRAFODION-2697] Set bulkload permissions for hbase upload
The /user/trafodion directory must be traversable for non-trafodion users.
Make bulkload directory readable by hbase group.
Bulkload directory should not be readable by other, to keep transient data secure.
For python, also do not create /lobs /bulkload dirctories, as they are no longer used.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/872b91cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/872b91cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/872b91cb
Branch: refs/heads/master
Commit: 872b91cb2155f51a383b77bc61daa5f0517c172c
Parents: 3ae998f
Author: Steve Varnau <sv...@apache.org>
Authored: Fri Jul 28 20:37:31 2017 +0000
Committer: Steve Varnau <sv...@apache.org>
Committed: Fri Jul 28 20:37:31 2017 +0000
----------------------------------------------------------------------
core/trafodion.spec | 16 +++++++++++++---
.../TRAFODION/2.1/package/scripts/params.py | 1 +
.../2.1/package/scripts/trafodionmaster.py | 9 ++++++++-
install/python-installer/scripts/hdfs_cmds.py | 8 ++++++--
4 files changed, 28 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/872b91cb/core/trafodion.spec
----------------------------------------------------------------------
diff --git a/core/trafodion.spec b/core/trafodion.spec
index 4ecc873..34ceed2 100644
--- a/core/trafodion.spec
+++ b/core/trafodion.spec
@@ -96,9 +96,19 @@ then
fi
if ! getent passwd trafodion > /dev/null
then
- /usr/sbin/useradd --shell /bin/bash -m trafodion -g trafodion -G hbase,hive --home /home/trafodion > /dev/null 2>&1
-else
- /usr/sbin/usermod -a -G hbase,hive trafodion > /dev/null 2>&1
+ /usr/sbin/useradd --shell /bin/bash -m trafodion -g trafodion --home /home/trafodion > /dev/null 2>&1
+fi
+if getent group hbase > /dev/null
+then
+ /usr/sbin/usermod -a -G hbase trafodion > /dev/null 2>&1
+fi
+if getent group hive > /dev/null
+then
+ /usr/sbin/usermod -a -G hive trafodion > /dev/null 2>&1
+fi
+if getent group hadoop > /dev/null
+then
+ /usr/sbin/usermod -a -G hadoop trafodion > /dev/null 2>&1
fi
chmod go+rx /home/trafodion
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/872b91cb/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py b/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py
index f1f34d4..8e4183b 100755
--- a/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py
+++ b/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/params.py
@@ -54,6 +54,7 @@ traf_clust_template = config['configurations']['traf-cluster-env']['content']
traf_user = 'trafodion'
traf_group = 'trafodion'
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
hbase_user = config['configurations']['hbase-env']['hbase_user']
hbase_staging = config['configurations']['hbase-site']['hbase.bulkload.staging.dir']
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/872b91cb/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py
----------------------------------------------------------------------
diff --git a/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py b/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py
index 331a206..4649d63 100755
--- a/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py
+++ b/install/ambari-installer/traf-mpack/common-services/TRAFODION/2.1/package/scripts/trafodionmaster.py
@@ -131,6 +131,12 @@ class Master(Script):
owner=params.hbase_user,
group=params.hbase_user,
)
+ params.HdfsDirectory("/user/trafodion",
+ action="create_on_execute",
+ owner=params.traf_user,
+ group=params.traf_group,
+ mode=0755,
+ )
params.HdfsDirectory("/user/trafodion/trafodion_backups",
action="create_on_execute",
owner=params.traf_user,
@@ -139,7 +145,8 @@ class Master(Script):
params.HdfsDirectory("/user/trafodion/bulkload",
action="create_on_execute",
owner=params.traf_user,
- group=params.traf_group,
+ group=params.user_group,
+ mode=0750,
)
params.HdfsDirectory("/user/trafodion/lobs",
action="create_on_execute",
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/872b91cb/install/python-installer/scripts/hdfs_cmds.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/hdfs_cmds.py b/install/python-installer/scripts/hdfs_cmds.py
index 1826f3b..12d342c 100755
--- a/install/python-installer/scripts/hdfs_cmds.py
+++ b/install/python-installer/scripts/hdfs_cmds.py
@@ -45,11 +45,15 @@ def run():
traf_user = dbcfgs['traf_user']
hdfs_user = dbcfgs['hdfs_user']
hbase_user = dbcfgs['hbase_user']
+ hbase_group = cmd_output('%s groups %s | cut -d" " -f3' % (hdfs_bin, hbase_user))
run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin)
- run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive' % (hdfs_bin, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /hbase/archive' % (hdfs_bin, traf_loc))
run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive' % (hdfs_bin, hbase_user, hbase_user))
- run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chmod 0755 %s' % (hdfs_bin, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chmod 0750 %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_loc))
+ run_cmd_as_user(hdfs_user, '%s dfs -chgrp %s %s/bulkload' % (hdfs_bin, hbase_group, traf_loc))
run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user))
run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin)
[3/4] incubator-trafodion git commit: [TRAFODION-2697] Fix couple
remaining references to hdfs:/bulkload
Posted by sv...@apache.org.
[TRAFODION-2697] Fix couple remaining references to hdfs:/bulkload
One regression test had a couple references missed in earlier change
to path of bulkload directory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/a9af6981
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/a9af6981
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/a9af6981
Branch: refs/heads/master
Commit: a9af6981462bc37b8321c58d626cf76e84845672
Parents: 250f60a
Author: Steve Varnau <sv...@apache.org>
Authored: Mon Jul 31 22:13:29 2017 +0000
Committer: Steve Varnau <sv...@apache.org>
Committed: Mon Jul 31 22:13:29 2017 +0000
----------------------------------------------------------------------
core/sql/regress/hive/EXPECTED018 | 4 ++--
core/sql/regress/hive/TEST018 | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a9af6981/core/sql/regress/hive/EXPECTED018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
index 79aa4bc..01eb789 100644
--- a/core/sql/regress/hive/EXPECTED018
+++ b/core/sql/regress/hive/EXPECTED018
@@ -530,7 +530,7 @@ cat /tmp/merged_customer_address | wc -l
>>--------------------------
>>--exp2
>>explain options 'f'
-+>UNLOAD EXTRACT TO '/bulkload/customer_demographics'
++>UNLOAD EXTRACT TO '/user/trafodion/bulkload/customer_demographics'
+>select * from trafodion.hbase.customer_demographics
+><<+ cardinality 10e10 >>;
@@ -614,7 +614,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>WITH PURGEDATA FROM TARGET
+>MERGE FILE 'merged_customer_demogs_3' OVERWRITE
+>--COMPRESSION GZIP
-+>INTO '/bulkload/customer_demographics_salt'
++>INTO '/user/trafodion/bulkload/customer_demographics_salt'
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a9af6981/core/sql/regress/hive/TEST018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018
index a5544c1..6b273ec 100644
--- a/core/sql/regress/hive/TEST018
+++ b/core/sql/regress/hive/TEST018
@@ -308,7 +308,7 @@ log LOG018;
--------------------------
--exp2
explain options 'f'
-UNLOAD EXTRACT TO '/bulkload/customer_demographics'
+UNLOAD EXTRACT TO '/user/trafodion/bulkload/customer_demographics'
select * from trafodion.hbase.customer_demographics
<<+ cardinality 10e10 >>;
--unload 2
@@ -351,7 +351,7 @@ UNLOAD
WITH PURGEDATA FROM TARGET
MERGE FILE 'merged_customer_demogs_3' OVERWRITE
--COMPRESSION GZIP
-INTO '/bulkload/customer_demographics_salt'
+INTO '/user/trafodion/bulkload/customer_demographics_salt'
select * from trafodion.hbase.customer_demographics_salt
<<+ cardinality 10e10 >>;
[2/4] incubator-trafodion git commit: [TRAFODION-2697] Make sure
trafodion is member of hbase, hive groups
Posted by sv...@apache.org.
[TRAFODION-2697] Make sure trafodion is member of hbase, hive groups
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/250f60ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/250f60ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/250f60ac
Branch: refs/heads/master
Commit: 250f60acb9b4a62fbdd581f6b3ab1da0e57806d9
Parents: 872b91c
Author: Steve Varnau <sv...@apache.org>
Authored: Fri Jul 28 20:51:11 2017 +0000
Committer: Steve Varnau <sv...@apache.org>
Committed: Fri Jul 28 20:51:11 2017 +0000
----------------------------------------------------------------------
install/python-installer/scripts/traf_user.py | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/250f60ac/install/python-installer/scripts/traf_user.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/traf_user.py b/install/python-installer/scripts/traf_user.py
index 60e191c..70f7ca3 100755
--- a/install/python-installer/scripts/traf_user.py
+++ b/install/python-installer/scripts/traf_user.py
@@ -69,6 +69,13 @@ def run():
run_cmd('groupadd %s > /dev/null 2>&1' % traf_group)
traf_pwd = dbcfgs['traf_pwd']
run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd))
+ # hbase group is generally either hbase or hadoop, depending on distro
+ if cmd_output('getent group hbase'):
+ cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user)
+ if cmd_output('getent group hadoop'):
+ cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user)
+ if cmd_output('getent group hive'):
+ cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user)
if not os.path.exists(traf_user_dir):
run_cmd('mkdir -p %s' % traf_user_dir)
[4/4] incubator-trafodion git commit: Merge [TRAFODION-2697] Set
bulkload permissions for hbase upload
Posted by sv...@apache.org.
Merge [TRAFODION-2697] Set bulkload permissions for hbase upload
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d86a0baf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d86a0baf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d86a0baf
Branch: refs/heads/master
Commit: d86a0baf3b9053eb220371cfeac847143216626b
Parents: ed8666d a9af698
Author: Steve Varnau <sv...@apache.org>
Authored: Wed Aug 2 16:37:26 2017 +0000
Committer: Steve Varnau <sv...@apache.org>
Committed: Wed Aug 2 16:37:26 2017 +0000
----------------------------------------------------------------------
core/sql/regress/hive/EXPECTED018 | 4 ++--
core/sql/regress/hive/TEST018 | 4 ++--
core/trafodion.spec | 16 +++++++++++++---
.../TRAFODION/2.1/package/scripts/params.py | 1 +
.../2.1/package/scripts/trafodionmaster.py | 9 ++++++++-
install/python-installer/scripts/hdfs_cmds.py | 8 ++++++--
install/python-installer/scripts/traf_user.py | 7 +++++++
7 files changed, 39 insertions(+), 10 deletions(-)
----------------------------------------------------------------------