You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2017/04/13 09:39:21 UTC
incubator-hawq git commit: HAWQ-1396. Add more test cases for
querying external table via PXF with Ranger enable.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 2d9bc0306 -> 72928dab2
HAWQ-1396. Add more test cases for querying external table via PXF with Ranger enable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/72928dab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/72928dab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/72928dab
Branch: refs/heads/master
Commit: 72928dab2dfc386118ae95e4016f3b549e347de1
Parents: 2d9bc03
Author: interma <in...@outlook.com>
Authored: Wed Apr 5 13:47:40 2017 +0800
Committer: hubertzhang <hu...@apache.org>
Committed: Thu Apr 13 17:38:29 2017 +0800
----------------------------------------------------------------------
src/test/feature/.gitignore | 2 +
.../feature/Ranger/ans/manual10000_success.ans | 10 +
.../feature/Ranger/ans/normal10000_success.ans | 10 -
src/test/feature/Ranger/ans/pxf1_fail.ans | 2 +-
src/test/feature/Ranger/ans/pxf2_fail.ans | 9 +
src/test/feature/Ranger/ans/pxf2_success.ans | 8 +
src/test/feature/Ranger/ans/pxf3_success.ans | 10 +
src/test/feature/Ranger/ans/pxf4_fail.ans | 9 +
src/test/feature/Ranger/ans/pxf4_success.ans | 8 +
src/test/feature/Ranger/ans/pxf5_success.ans | 13 +
src/test/feature/Ranger/ans/pxf6_fail.ans | 8 +
src/test/feature/Ranger/ans/pxf6_success.ans | 8 +
src/test/feature/Ranger/ans/pxf7_success.ans | 14 +
src/test/feature/Ranger/ans/pxf8_fail.ans | 9 +
src/test/feature/Ranger/ans/pxf8_success.ans | 8 +
src/test/feature/Ranger/ans/pxf9_success.ans | 12 +
.../Ranger/data/testhive_externaltable.sql | 5 +
src/test/feature/Ranger/policy/10000/1.json | 1 -
src/test/feature/Ranger/policy_helper.cpp | 204 +++++++++++++
src/test/feature/Ranger/policy_helper.h | 110 +++++++
src/test/feature/Ranger/pxfpolicy/1/3.json | 2 +-
src/test/feature/Ranger/rangerpolicy.py | 56 ++--
src/test/feature/Ranger/rangerrest.py | 1 +
src/test/feature/Ranger/sql/manual/10000.sql | 2 +-
src/test/feature/Ranger/sql/pxf/2.sql | 4 +
src/test/feature/Ranger/sql/pxf/3.sql | 4 +
src/test/feature/Ranger/sql/pxf/4.sql | 4 +
src/test/feature/Ranger/sql/pxf/5.sql | 3 +
src/test/feature/Ranger/sql/pxf/6.sql | 4 +
src/test/feature/Ranger/sql/pxf/7.sql | 3 +
src/test/feature/Ranger/sql/pxf/8.sql | 4 +
src/test/feature/Ranger/sql/pxf/9.sql | 3 +
src/test/feature/Ranger/test_policyhelper.cpp | 92 ++++++
src/test/feature/Ranger/test_ranger.cpp | 301 ++++++++++++++++---
src/test/feature/Ranger/test_ranger.h | 7 +-
src/test/feature/sanity_tests.txt | 4 +-
36 files changed, 873 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/.gitignore
----------------------------------------------------------------------
diff --git a/src/test/feature/.gitignore b/src/test/feature/.gitignore
index fec82bd..10fc2f6 100644
--- a/src/test/feature/.gitignore
+++ b/src/test/feature/.gitignore
@@ -23,3 +23,5 @@ testlib/ans/template.ans
testlib/sql/template.sql
utility/ans/copytest.csv
utility/ans/onek.data
+
+*.pyc
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/manual10000_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/manual10000_success.ans b/src/test/feature/Ranger/ans/manual10000_success.ans
new file mode 100644
index 0000000..73f878b
--- /dev/null
+++ b/src/test/feature/Ranger/ans/manual10000_success.ans
@@ -0,0 +1,10 @@
+-- start_ignore
+-- end_ignore
+set session role=usermanual10000;
+SET
+select count(*) from information_schema.view_table_usage;
+0
+select count(*) from hawq_toolkit.hawq_table_indexes;
+0
+select count(*) from pg_catalog.pg_compression;
+4
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/normal10000_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/normal10000_success.ans b/src/test/feature/Ranger/ans/normal10000_success.ans
deleted file mode 100644
index 125e649..0000000
--- a/src/test/feature/Ranger/ans/normal10000_success.ans
+++ /dev/null
@@ -1,10 +0,0 @@
--- start_ignore
--- end_ignore
-set session role=usertest10000;
-SET
-select count(*) from information_schema.view_table_usage;
-0
-select count(*) from hawq_toolkit.hawq_table_indexes;
-0
-select count(*) from pg_catalog.pg_compression;
-4
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf1_fail.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf1_fail.ans b/src/test/feature/Ranger/ans/pxf1_fail.ans
index a8a7197..fb2e460 100644
--- a/src/test/feature/Ranger/ans/pxf1_fail.ans
+++ b/src/test/feature/Ranger/ans/pxf1_fail.ans
@@ -5,6 +5,6 @@ SET
set session role= 'userpxf1';
SET
select * from hcatalog.default.testhive;
-psql:/tmp/TestHawqRanger_HcatalogTest.sql:5: ERROR: permission denied for schema default
+psql:/tmp/TestHawqRanger_PXFHcatalogTest.sql:5: ERROR: permission denied for schema default
LINE 1: select * from hcatalog.default.testhive;
^
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf2_fail.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf2_fail.ans b/src/test/feature/Ranger/ans/pxf2_fail.ans
new file mode 100644
index 0000000..ff7c32b
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf2_fail.ans
@@ -0,0 +1,9 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf2';
+SET
+CREATE WRITABLE EXTERNAL TABLE pxf_hdfs_writabletbl_1(location text, month text, num_orders int, total_sales float8)
+ LOCATION ('pxf://localhost:51200/ranger_test/pxfwritable_hdfs_textsimple1?PROFILE=HdfsTextSimple')
+ FORMAT 'TEXT' (delimiter=E',');
+psql:/tmp/TestHawqRanger_PXFHDFSTest.sql:7: WARNING: usage privilege of namespace testhawqranger_pxfhdfstest is required.
+psql:/tmp/TestHawqRanger_PXFHDFSTest.sql:7: ERROR: permission denied for external protocol pxf
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf2_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf2_success.ans b/src/test/feature/Ranger/ans/pxf2_success.ans
new file mode 100644
index 0000000..1f37ff8
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf2_success.ans
@@ -0,0 +1,8 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf2';
+SET
+CREATE WRITABLE EXTERNAL TABLE pxf_hdfs_writabletbl_1(location text, month text, num_orders int, total_sales float8)
+ LOCATION ('pxf://localhost:51200/ranger_test/pxfwritable_hdfs_textsimple1?PROFILE=HdfsTextSimple')
+ FORMAT 'TEXT' (delimiter=E',');
+CREATE EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf3_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf3_success.ans b/src/test/feature/Ranger/ans/pxf3_success.ans
new file mode 100644
index 0000000..d6ccbba
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf3_success.ans
@@ -0,0 +1,10 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf2';
+SET
+INSERT INTO pxf_hdfs_writabletbl_1 VALUES ( 'Frankfurt', 'Mar', 777, 3956.98 );
+INSERT 0 1
+INSERT INTO pxf_hdfs_writabletbl_1 VALUES ( 'Cleveland', 'Oct', 3812, 96645.37 );
+INSERT 0 1
+drop external table pxf_hdfs_writabletbl_1;
+DROP EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf4_fail.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf4_fail.ans b/src/test/feature/Ranger/ans/pxf4_fail.ans
new file mode 100644
index 0000000..b7562b3
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf4_fail.ans
@@ -0,0 +1,9 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf3';
+SET
+CREATE EXTERNAL TABLE pxf_hdfs_textsimple_r1(location text, month text, num_orders int, total_sales float8)
+ LOCATION ('pxf://localhost:51200/ranger_test/pxfwritable_hdfs_textsimple1?PROFILE=HdfsTextSimple')
+ FORMAT 'CSV';
+psql:/tmp/TestHawqRanger_PXFHDFSTest.sql:7: WARNING: usage privilege of namespace testhawqranger_pxfhdfstest is required.
+psql:/tmp/TestHawqRanger_PXFHDFSTest.sql:7: ERROR: permission denied for external protocol pxf
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf4_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf4_success.ans b/src/test/feature/Ranger/ans/pxf4_success.ans
new file mode 100644
index 0000000..5bbe124
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf4_success.ans
@@ -0,0 +1,8 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf3';
+SET
+CREATE EXTERNAL TABLE pxf_hdfs_textsimple_r1(location text, month text, num_orders int, total_sales float8)
+ LOCATION ('pxf://localhost:51200/ranger_test/pxfwritable_hdfs_textsimple1?PROFILE=HdfsTextSimple')
+ FORMAT 'CSV';
+CREATE EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf5_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf5_success.ans b/src/test/feature/Ranger/ans/pxf5_success.ans
new file mode 100644
index 0000000..89f744c
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf5_success.ans
@@ -0,0 +1,13 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf3';
+SET
+SELECT * FROM pxf_hdfs_textsimple_r1;
+ location | month | num_orders | total_sales
+-----------+-------+------------+-------------
+ Cleveland | Oct | 3812 | 96645.37
+ Frankfurt | Mar | 777 | 3956.98
+(2 rows)
+
+drop external table pxf_hdfs_textsimple_r1;
+DROP EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf6_fail.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf6_fail.ans b/src/test/feature/Ranger/ans/pxf6_fail.ans
new file mode 100644
index 0000000..cced8c2
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf6_fail.ans
@@ -0,0 +1,8 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf4';
+SET
+CREATE EXTERNAL TABLE testhive_ext(a int, b int)
+ LOCATION ('pxf://localhost:51200/default.testhive_ext?PROFILE=Hive')
+ FORMAT 'custom' (formatter='pxfwritable_import');
+psql:/tmp/TestHawqRanger_PXFHiveTest.sql:7: ERROR: permission denied for external protocol pxf
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf6_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf6_success.ans b/src/test/feature/Ranger/ans/pxf6_success.ans
new file mode 100644
index 0000000..e808202
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf6_success.ans
@@ -0,0 +1,8 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf4';
+SET
+CREATE EXTERNAL TABLE testhive_ext(a int, b int)
+ LOCATION ('pxf://localhost:51200/default.testhive_ext?PROFILE=Hive')
+ FORMAT 'custom' (formatter='pxfwritable_import');
+CREATE EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf7_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf7_success.ans b/src/test/feature/Ranger/ans/pxf7_success.ans
new file mode 100644
index 0000000..b58cc54
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf7_success.ans
@@ -0,0 +1,14 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf4';
+SET
+select * from testhive_ext;
+ a | b
+---+---
+ 1 | 2
+ 2 | 4
+ 3 | 6
+(3 rows)
+
+drop external table testhive_ext;
+DROP EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf8_fail.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf8_fail.ans b/src/test/feature/Ranger/ans/pxf8_fail.ans
new file mode 100644
index 0000000..8318711
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf8_fail.ans
@@ -0,0 +1,9 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf5';
+SET
+CREATE EXTERNAL TABLE test_hbase (recordkey bytea,"f1:col1" int)
+ LOCATION ('pxf://localhost:51200/test_hbase?Profile=HBase')
+ FORMAT 'CUSTOM' (Formatter='pxfwritable_import');
+psql:/tmp/TestHawqRanger_PXFHBaseTest.sql:7: WARNING: usage privilege of namespace testhawqranger_pxfhbasetest is required.
+psql:/tmp/TestHawqRanger_PXFHBaseTest.sql:7: ERROR: permission denied for external protocol pxf
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf8_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf8_success.ans b/src/test/feature/Ranger/ans/pxf8_success.ans
new file mode 100644
index 0000000..bbf901a
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf8_success.ans
@@ -0,0 +1,8 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf5';
+SET
+CREATE EXTERNAL TABLE test_hbase (recordkey bytea,"f1:col1" int)
+ LOCATION ('pxf://localhost:51200/test_hbase?Profile=HBase')
+ FORMAT 'CUSTOM' (Formatter='pxfwritable_import');
+CREATE EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/ans/pxf9_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/pxf9_success.ans b/src/test/feature/Ranger/ans/pxf9_success.ans
new file mode 100644
index 0000000..599ead7
--- /dev/null
+++ b/src/test/feature/Ranger/ans/pxf9_success.ans
@@ -0,0 +1,12 @@
+-- start_ignore
+-- end_ignore
+set session role= 'userpxf5';
+SET
+select * from test_hbase;
+ recordkey | f1:col1
+-----------+---------
+ r1 | 100
+(1 row)
+
+drop EXTERNAL TABLE test_hbase;
+DROP EXTERNAL TABLE
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/data/testhive_externaltable.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/data/testhive_externaltable.sql b/src/test/feature/Ranger/data/testhive_externaltable.sql
new file mode 100644
index 0000000..3727ec1
--- /dev/null
+++ b/src/test/feature/Ranger/data/testhive_externaltable.sql
@@ -0,0 +1,5 @@
+drop table if exists testhive_ext;
+CREATE TABLE testhive_ext (a int, b int) ;
+INSERT INTO testhive_ext VALUES(1, 2);
+INSERT INTO testhive_ext VALUES(2, 4);
+INSERT INTO testhive_ext VALUES(3, 6);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/policy/10000/1.json
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/policy/10000/1.json b/src/test/feature/Ranger/policy/10000/1.json
deleted file mode 100644
index e3a4d86..0000000
--- a/src/test/feature/Ranger/policy/10000/1.json
+++ /dev/null
@@ -1 +0,0 @@
-{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], "description": "no description", "isAuditEnabled": true, "isEnabled": true, "name": "policy10000-1", "policyItems": [{"accesses": [{"isAllowed": true, "type": "usage-schema"}, {"isAllowed": true, "type": "create"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": ["usertest10000", "usersuper10000"]}], "resources": {"database": {"isExcludes": false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": false, "values": ["public"]}, "table": {"isExcludes": false, "isRecursive": false, "values": ["*"]}}, "service": "hawq", "version": 1}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/policy_helper.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/policy_helper.cpp b/src/test/feature/Ranger/policy_helper.cpp
new file mode 100644
index 0000000..277a203
--- /dev/null
+++ b/src/test/feature/Ranger/policy_helper.cpp
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include "policy_helper.h"
+#include "lib/command.h"
+#include "lib/string_util.h"
+
+using namespace std;
+
+#define TMP_FOLDER "/tmp/ranger_test/"
+
+
+PolicyHelper::PolicyHelper (string root_path, string ranger_host): _root_path (root_path), _rangerhost(ranger_host)
+{}
+
+PolicyHelper::~PolicyHelper ()
+{}
+
+void PolicyHelper::Reset()
+{
+ _policys.clear();
+}
+
+int PolicyHelper::add_common_tsf(string policy_name, string user,
+ string database, string schema, string tsf_field, string tsf_value,
+ vector<string> accesses)
+{
+
+ const char * policy_template =
+ "{\
+ \"allowExceptions\": [ ], \
+ \"denyExceptions\": [ ], \
+ \"denyPolicyItems\": [ ], \
+ \"description\": \"no description\", \
+ \"isAuditEnabled\": true, \
+ \"isEnabled\": true, \
+ \"name\": \"%s\", \
+ \"policyItems\": [\
+ {\
+ \"accesses\": [ %s ], \
+ \"conditions\": [ ], \
+ \"delegateAdmin\": true, \
+ \"groups\": null, \
+ \"users\": [\"%s\"]\
+ }\
+ ], \
+ \"resources\": {\
+ \"database\": {\
+ \"isExcludes\": false, \
+ \"isRecursive\": false, \
+ \"values\": [\"%s\"]\
+ }, \
+ \"schema\": {\
+ \"isExcludes\": false, \
+ \"isRecursive\": false, \
+ \"values\": [\"%s\"]\
+ }, \
+ \"%s\": {\
+ \"isExcludes\": false, \
+ \"isRecursive\": false, \
+ \"values\": [\"%s\"]\
+ }\
+ }, \
+ \"service\": \"hawq\", \
+ \"version\": 3\
+ }";
+
+ string accesses_str;
+ for (auto access : accesses)
+ {
+ const char * access_template = "{\"isAllowed\": true, \"type\": \"%s\"}";
+ string acc_str = hawq::test::stringFormat(access_template, access.c_str());
+ if (accesses_str != "")
+ {
+ accesses_str += ",";
+ }
+ accesses_str += acc_str;
+ }
+
+ string policy_str = hawq::test::stringFormat(policy_template,
+ policy_name.c_str(), accesses_str.c_str(), user.c_str(),
+ database.c_str(), schema.c_str(), tsf_field.c_str(), tsf_value.c_str());
+
+ _policys.push_back(policy_str);
+ //cout<<policy_str<<endl;
+
+ return 0;
+}
+
+int PolicyHelper::add_common_sp(std::string policy_name, std::string user,
+ std::string tsp_field, std::string tsp_value,
+ std::vector<std::string> accesses)
+{
+ const char * policy_template =
+ "{\
+ \"allowExceptions\": [ ], \
+ \"denyExceptions\": [ ], \
+ \"denyPolicyItems\": [ ], \
+ \"description\": \"no description\", \
+ \"isAuditEnabled\": true, \
+ \"isEnabled\": true, \
+ \"name\": \"%s\", \
+ \"policyItems\": [\
+ {\
+ \"accesses\": [ %s ], \
+ \"conditions\": [ ], \
+ \"delegateAdmin\": true, \
+ \"groups\": null, \
+ \"users\": [\"%s\"]\
+ }\
+ ], \
+ \"resources\": {\
+ \"%s\": {\
+ \"isExcludes\": false, \
+ \"isRecursive\": false, \
+ \"values\": [\"%s\"]\
+ }\
+ }, \
+ \"service\": \"hawq\", \
+ \"version\": 3\
+ }";
+
+ string accesses_str;
+ for (auto access : accesses)
+ {
+ const char * access_template = "{\"isAllowed\": true, \"type\": \"%s\"}";
+ string acc_str = hawq::test::stringFormat(access_template, access.c_str());
+ if (accesses_str != "")
+ {
+ accesses_str += ",";
+ }
+ accesses_str += acc_str;
+ }
+
+ string policy_str = hawq::test::stringFormat(policy_template,
+ policy_name.c_str(), accesses_str.c_str(), user.c_str(), tsp_field.c_str(), tsp_value.c_str());
+
+ _policys.push_back(policy_str);
+ return 0;
+}
+
+string PolicyHelper::write_tmpfile(std::string content)
+{
+ char temp[] = TMP_FOLDER "fileXXXXXX";
+ const char *file_name = mktemp(temp);
+
+ int fd = open(file_name, O_CREAT | O_TRUNC | O_WRONLY);
+ if (fd <= 0)
+ return "";
+ write(fd, content.c_str(), content.length());
+ close(fd);
+ return string(file_name);
+}
+
+int PolicyHelper::ActivateAllPoliciesOnRanger()
+{
+ auto cmd = "mkdir -p " TMP_FOLDER;
+ hawq::test::Command::getCommandStatus(cmd);
+ for (auto policy : _policys)
+ {
+ string file_path= write_tmpfile(policy);
+ if (file_path == "")
+ return -1;
+ auto cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -a %s",
+ _root_path.c_str(), _rangerhost.c_str(), file_path.c_str() );
+ int ret = hawq::test::Command::getCommandStatus(cmd);
+ if (ret != 0)
+ return -1;
+ }
+ sleep(60);
+ return 0;
+}
+
+int PolicyHelper::DeletePolicyOnRanger(std::string policy_name)
+{
+ auto cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -d %s",
+ _root_path.c_str(), _rangerhost.c_str(), policy_name.c_str() );
+ int ret = hawq::test::Command::getCommandStatus(cmd);
+ if (ret != 0)
+ return -1;
+ return 0;
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/policy_helper.h
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/policy_helper.h b/src/test/feature/Ranger/policy_helper.h
new file mode 100644
index 0000000..ce990ad
--- /dev/null
+++ b/src/test/feature/Ranger/policy_helper.h
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef _POLICY_HELPER_H_
+#define _POLICY_HELPER_H_
+
+#include <string>
+#include <vector>
+
+
+/**
+ * A helper class for ranger policy add/del, using rangerpolicy.py internally
+ * Usage see test_policyhelper.cpp
+ */
+class PolicyHelper{
+public:
+ PolicyHelper (std::string root_path, std::string ranger_host);
+ ~PolicyHelper ();
+ void Reset();
+
+ /*
+ * send all policies to ranger and sleep 60s for taking effect
+ * @returns: 0 success; -1 failed
+ */
+ int ActivateAllPoliciesOnRanger();
+
+ /*
+ Resources hierarchy:
+ |--database
+ | |--schema
+ | | |--table
+ | | |--sequence
+ | | |--function
+ | |
+ | |--language
+ |
+ |--tablespace
+ |--protocol
+ */
+ int AddSchemaPolicy(std::string policy_name, std::string user,
+ std::string database, std::string schema, std::vector<std::string> accesses)
+ {
+ return add_common_tsf(policy_name, user, database, schema, "table", "*", accesses);
+ }
+
+ int AddTablePolicy(std::string policy_name, std::string user,
+ std::string database, std::string schema, std::string table, std::vector<std::string> accesses)
+ {
+ return add_common_tsf(policy_name, user, database, schema, "table", table, accesses);
+ }
+
+ int AddProtocolPolicy(std::string policy_name, std::string user,
+ std::string protocol, std::vector<std::string> accesses)
+ {
+ return add_common_sp(policy_name, user, "protocol", protocol, accesses);
+ }
+
+ //TODO other AddXXXPolicy function
+
+ int DeletePolicyOnRanger(std::string user);
+
+private:
+ std::string _rangerhost;
+ std::string _root_path;
+
+ std::vector<std::string> _policys;
+
+ std::string write_tmpfile(std::string content);
+ int add_common_t(std::string policy_name, std::string user,
+ std::string database, std::string schema, std::string tsf_field, std::string tsf_value,
+ std::vector<std::string> accesses);
+
+ /**
+ * tsf means:
+ * t: table
+ * s: schema
+ * f: function
+ */
+ int add_common_tsf(std::string policy_name, std::string user,
+ std::string database, std::string schema, std::string tsf_field, std::string tsf_value,
+ std::vector<std::string> accesses);
+
+ /**
+ * sp means:
+ * s: tablespace
+ * p: protocol
+ */
+ int add_common_sp(std::string policy_name, std::string user,
+ std::string tsp_field, std::string tsp_value,
+ std::vector<std::string> accesses);
+};
+
+#endif
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/pxfpolicy/1/3.json
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/pxfpolicy/1/3.json b/src/test/feature/Ranger/pxfpolicy/1/3.json
index 08937c4..d04dcc5 100644
--- a/src/test/feature/Ranger/pxfpolicy/1/3.json
+++ b/src/test/feature/Ranger/pxfpolicy/1/3.json
@@ -1 +1 @@
-{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], "description": "no description", "isAuditEnabled": true, "isEnabled": true, "name": "pxfpolicy1-3", "policyItems": [{"accesses": [{"isAllowed": true, "type": "usage-schema"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": ["userpxf1"]}], "resources": {"database": {"isExcludes": false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": false, "values": ["testhawqranger_hcatalogtest"]}, "table": {"isExcludes": false, "isRecursive": false, "values": ["*"]}}, "service": "hawq", "version": 3}
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], "description": "no description", "isAuditEnabled": true, "isEnabled": true, "name": "pxfpolicy1-3", "policyItems": [{"accesses": [{"isAllowed": true, "type": "usage-schema"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": ["userpxf1"]}], "resources": {"database": {"isExcludes": false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": false, "values": ["testhawqranger_pxfhcatalogtest"]}, "table": {"isExcludes": false, "isRecursive": false, "values": ["*"]}}, "service": "hawq", "version": 3}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/rangerpolicy.py
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/rangerpolicy.py b/src/test/feature/Ranger/rangerpolicy.py
index b5a7c48..cfc4d66 100644
--- a/src/test/feature/Ranger/rangerpolicy.py
+++ b/src/test/feature/Ranger/rangerpolicy.py
@@ -62,26 +62,30 @@ def create_policy(policy_json_file_name, rangerhelper):
policy_end_pos = response.find("], service=[")
dup_policy_name = response[0:policy_end_pos]
- #get dupulicate policy and add privilege item.
+ #get duplicate policy and add privilege item.
service_name = 'hawq'
- print dup_policy_name;
- response, is_success = rangerhelper.get_policy(service_name, dup_policy_name);
- response_dict = json.load(response)
- for new_policy_item in json_decode['policyItems']:
- response_dict["policyItems"].append(new_policy_item)
- for new_policy_item in json_decode['denyPolicyItems']:
- response_dict["denyPolicyItems"].append(new_policy_item)
- for new_policy_item in json_decode['allowExceptions']:
- response_dict["allowExceptions"].append(new_policy_item)
- for new_policy_item in json_decode['denyExceptions']:
- response_dict["denyExceptions"].append(new_policy_item)
+ print "find duplicate policy, try to update [%s]" % (dup_policy_name)
+ response, is_success = rangerhelper.get_policy(service_name, dup_policy_name)
+ if is_success:
+ response_dict = json.load(response)
+ for new_policy_item in json_decode['policyItems']:
+ response_dict["policyItems"].append(new_policy_item)
+ for new_policy_item in json_decode['denyPolicyItems']:
+ response_dict["denyPolicyItems"].append(new_policy_item)
+ for new_policy_item in json_decode['allowExceptions']:
+ response_dict["allowExceptions"].append(new_policy_item)
+ for new_policy_item in json_decode['denyExceptions']:
+ response_dict["denyExceptions"].append(new_policy_item)
+ response, is_success = rangerhelper.update_policy(service_name, dup_policy_name, \
+ json.dumps(response_dict))
+ else:
+ return policyname, False
- rangerhelper.update_policy(service_name, dup_policy_name, \
- json.dumps(response_dict));
- return policyname
+ return policyname, is_success
def delete_policy(delete_policy_name, rangerhelper):
- rangerhelper.delete_policy("hawq", delete_policy_name);
+ response, is_success = rangerhelper.delete_policy("hawq", delete_policy_name)
+ return is_success
if __name__ == '__main__':
@@ -96,12 +100,22 @@ if __name__ == '__main__':
delete_policy_name = options.deletedpolicyname
#init rangerresthelper
- helper = RangerRestHelper(host, port, rangeruser, rangerpasswd);
+ helper = RangerRestHelper(host, port, rangeruser, rangerpasswd)
if new_policy_json_file_name != "":
- policyname = create_policy(new_policy_json_file_name, helper)
- print "policy {} created".format(policyname)
+ policyname, is_success = create_policy(new_policy_json_file_name, helper)
+ if is_success:
+ print "policy {} created".format(policyname)
+ else:
+ print "policy {} create failed".format(policyname)
+ sys.exit(-1)
if delete_policy_name != "":
- delete_policy(delete_policy_name, helper)
- print "policy {} deleted".format(delete_policy_name)
+ is_success = delete_policy(delete_policy_name, helper)
+ if is_success:
+ print "policy {} deleted".format(delete_policy_name)
+ else:
+ print "policy {} delete failed".format(delete_policy_name)
+ sys.exit(-1)
+
+ sys.exit(0)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/rangerrest.py
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/rangerrest.py b/src/test/feature/Ranger/rangerrest.py
index 851cdd8..fa3196a 100644
--- a/src/test/feature/Ranger/rangerrest.py
+++ b/src/test/feature/Ranger/rangerrest.py
@@ -46,6 +46,7 @@ class RangerRestHelper(object):
error_message = e.read()
print error_message
return error_message, False
+ return "HTTPError", False
def get_policy(self, service_name, policy_name):
url = 'http://' + self.host + ':' + self.port + '/service/public/v2/api/service/' + \
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/manual/10000.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/manual/10000.sql b/src/test/feature/Ranger/sql/manual/10000.sql
index 3912cb3..e217179 100644
--- a/src/test/feature/Ranger/sql/manual/10000.sql
+++ b/src/test/feature/Ranger/sql/manual/10000.sql
@@ -1,4 +1,4 @@
-set session role=usertest10000;
+set session role=usermanual10000;
select count(*) from information_schema.view_table_usage;
select count(*) from hawq_toolkit.hawq_table_indexes;
select count(*) from pg_catalog.pg_compression;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/2.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/2.sql b/src/test/feature/Ranger/sql/pxf/2.sql
new file mode 100644
index 0000000..ba8483b
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/2.sql
@@ -0,0 +1,4 @@
+set session role= 'userpxf2';
+CREATE WRITABLE EXTERNAL TABLE pxf_hdfs_writabletbl_1(location text, month text, num_orders int, total_sales float8)
+ LOCATION ('pxf://localhost:51200/ranger_test/pxfwritable_hdfs_textsimple1?PROFILE=HdfsTextSimple')
+ FORMAT 'TEXT' (delimiter=E',');
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/3.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/3.sql b/src/test/feature/Ranger/sql/pxf/3.sql
new file mode 100644
index 0000000..10b1938
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/3.sql
@@ -0,0 +1,4 @@
+set session role= 'userpxf2';
+INSERT INTO pxf_hdfs_writabletbl_1 VALUES ( 'Frankfurt', 'Mar', 777, 3956.98 );
+INSERT INTO pxf_hdfs_writabletbl_1 VALUES ( 'Cleveland', 'Oct', 3812, 96645.37 );
+drop external table pxf_hdfs_writabletbl_1;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/4.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/4.sql b/src/test/feature/Ranger/sql/pxf/4.sql
new file mode 100644
index 0000000..20e3728
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/4.sql
@@ -0,0 +1,4 @@
+set session role= 'userpxf3';
+CREATE EXTERNAL TABLE pxf_hdfs_textsimple_r1(location text, month text, num_orders int, total_sales float8)
+ LOCATION ('pxf://localhost:51200/ranger_test/pxfwritable_hdfs_textsimple1?PROFILE=HdfsTextSimple')
+ FORMAT 'CSV';
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/5.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/5.sql b/src/test/feature/Ranger/sql/pxf/5.sql
new file mode 100644
index 0000000..b022fb2
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/5.sql
@@ -0,0 +1,3 @@
+set session role= 'userpxf3';
+SELECT * FROM pxf_hdfs_textsimple_r1;
+drop external table pxf_hdfs_textsimple_r1;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/6.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/6.sql b/src/test/feature/Ranger/sql/pxf/6.sql
new file mode 100644
index 0000000..50cdbbe
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/6.sql
@@ -0,0 +1,4 @@
+set session role= 'userpxf4';
+CREATE EXTERNAL TABLE testhive_ext(a int, b int)
+ LOCATION ('pxf://localhost:51200/default.testhive_ext?PROFILE=Hive')
+ FORMAT 'custom' (formatter='pxfwritable_import');
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/7.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/7.sql b/src/test/feature/Ranger/sql/pxf/7.sql
new file mode 100644
index 0000000..28eb39e
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/7.sql
@@ -0,0 +1,3 @@
+set session role= 'userpxf4';
+select * from testhive_ext;
+drop external table testhive_ext;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/8.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/8.sql b/src/test/feature/Ranger/sql/pxf/8.sql
new file mode 100644
index 0000000..71a8e76
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/8.sql
@@ -0,0 +1,4 @@
+set session role= 'userpxf5';
+CREATE EXTERNAL TABLE test_hbase (recordkey bytea,"f1:col1" int)
+ LOCATION ('pxf://localhost:51200/test_hbase?Profile=HBase')
+ FORMAT 'CUSTOM' (Formatter='pxfwritable_import');
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/sql/pxf/9.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/pxf/9.sql b/src/test/feature/Ranger/sql/pxf/9.sql
new file mode 100644
index 0000000..6d8b3b9
--- /dev/null
+++ b/src/test/feature/Ranger/sql/pxf/9.sql
@@ -0,0 +1,3 @@
+set session role= 'userpxf5';
+select * from test_hbase;
+drop EXTERNAL TABLE test_hbase;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/test_policyhelper.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/test_policyhelper.cpp b/src/test/feature/Ranger/test_policyhelper.cpp
new file mode 100644
index 0000000..b2c55af
--- /dev/null
+++ b/src/test/feature/Ranger/test_policyhelper.cpp
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "policy_helper.h"
+#include "gtest/gtest.h"
+#include "lib/sql_util.h"
+
+using namespace std;
+
+class TestRangerPolicyHelper : public ::testing::Test {
+public:
+ TestRangerPolicyHelper()
+ {
+ _rangerHost = RANGER_HOST;
+ }
+ string _rangerHost;
+ string _suffix = "PolicyHelper_";
+};
+
+TEST_F(TestRangerPolicyHelper, BasicTest) {
+ hawq::test::SQLUtility util;
+ PolicyHelper helper(util.getTestRootPath(), _rangerHost);
+ int ret = 0;
+ string user = _suffix+"TestUser"; //Note: must create this user on Ranger Admin first
+ string database = _suffix+"Testdb";
+ string table = _suffix+"TestTable";
+ string schema = "public";
+ string protocol= "protocol";
+ std::vector<std::string> accesses;
+
+ ret = helper.AddSchemaPolicy(_suffix+"Policy1", user, database, schema, {"usage-schema", "create-schema"});
+ EXPECT_EQ(0,ret);
+
+ accesses.clear();
+ accesses.push_back("select");
+ ret = helper.AddTablePolicy(_suffix+"Policy2", user, database, schema, table, accesses);
+ EXPECT_EQ(0,ret);
+
+ accesses.clear();
+ accesses.push_back("select");
+ ret = helper.AddProtocolPolicy(_suffix+"Policy3", user, protocol, accesses);
+ EXPECT_EQ(0,ret);
+
+ ret = helper.ActivateAllPoliciesOnRanger();
+ EXPECT_EQ(0,ret);
+
+ ret = helper.DeletePolicyOnRanger(_suffix+"Policy1");
+ ret = helper.DeletePolicyOnRanger(_suffix+"Policy2");
+ ret = helper.DeletePolicyOnRanger(_suffix+"Policy3");
+ EXPECT_EQ(0,ret);
+}
+
+
+TEST_F(TestRangerPolicyHelper, DuplicateResource) {
+ hawq::test::SQLUtility util;
+ PolicyHelper helper(util.getTestRootPath(), _rangerHost);
+ int ret = 0;
+ string user = _suffix+"TestUser"; //Note: must create this user on Ranger Admin first
+ string user2 = _suffix+"TestUser2"; //Note: must create this user on Ranger Admin first
+ string database = _suffix+"Testdb";
+ string schema = "public";
+
+ ret = helper.AddSchemaPolicy(_suffix+"Policy1", user, database, schema, {"usage-schema"});
+ EXPECT_EQ(0,ret);
+ ret = helper.AddSchemaPolicy(_suffix+"Policy2", user2, database, schema, {"create-schema"});
+ EXPECT_EQ(0,ret);
+
+ ret = helper.ActivateAllPoliciesOnRanger(); //policy2's access combine into policy1
+ EXPECT_EQ(0,ret);
+
+ ret = helper.DeletePolicyOnRanger(_suffix+"Policy1");
+ EXPECT_EQ(0,ret);
+ ret = helper.DeletePolicyOnRanger(_suffix+"Policy2");
+ EXPECT_EQ(-1,ret);
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/test_ranger.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/test_ranger.cpp b/src/test/feature/Ranger/test_ranger.cpp
index 9a61826..b386956 100644
--- a/src/test/feature/Ranger/test_ranger.cpp
+++ b/src/test/feature/Ranger/test_ranger.cpp
@@ -18,6 +18,7 @@
*/
#include "test_ranger.h"
+#include "policy_helper.h"
#include <string>
#include <pwd.h>
@@ -123,7 +124,6 @@ TEST_F(TestHawqRanger, BasicTest) {
string super_ansfile_success = hawq::test::stringFormat("Ranger/ans/super%d_success.ans", i);
cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), i);
- int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str());
cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str(), i);
int supersqlexist = std::atoi(Command::getCommandOutput(cmd).c_str());
@@ -179,39 +179,31 @@ static void clear_env(SQLUtility &util, int sql_id, string rootPath, string rang
TEST_F(TestHawqRanger, FallbackTest) {
SQLUtility util;
+ PolicyHelper helper(util.getTestRootPath(), rangerHost);
if (util.getGUCValue("hawq_acl_type") == "ranger")
{
- string rootPath(util.getTestRootPath());
- string rangerHost = RANGER_HOST;
- string initfile = "Ranger/sql/init_file";
- string cmd;
- int FallbackCase = 10000;
- int i = FallbackCase;
-
- // clear environment
- clear_env(util, i, rootPath, rangerHost);
-
- // create user_num
- std::string normalusername = hawq::test::stringFormat("usertest%d", i);;
- std::string superusername = hawq::test::stringFormat("usersuper%d", i);;
- util.execute(hawq::test::stringFormat("create role %s with login createdb;", normalusername.c_str()),true);
- util.execute(hawq::test::stringFormat("create role %s with login createdb superuser;", superusername.c_str()),true);
- // add user
- cmd = hawq::test::stringFormat("python %s/Ranger/rangeruser.py -h %s -u %s,%s", rootPath.c_str(),
- rangerHost.c_str(), normalusername.c_str(), superusername.c_str());
- Command::getCommandStatus(cmd);
- // add policy
- cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -a %s/Ranger/policy/%d/%d.json", rootPath.c_str(), rangerHost.c_str(), rootPath.c_str(), i, 1);
- Command::getCommandStatus(cmd);
- sleep(60);
-
- // run sql test
- string normal_sqlfile = hawq::test::stringFormat("Ranger/sql/manual/%d.sql", i);
- string normal_ansfile_success = hawq::test::stringFormat("Ranger/ans/normal%d_success.ans", i);
- util.execSQLFile(normal_sqlfile, normal_ansfile_success, initfile, true, true);
-
- clear_env(util, i, rootPath, rangerHost);
+ int idx = 10000;
+ int ret = 0;
+ const char * prefix = "manual";
+ std::string db = "hawq_feature_test_db";
+ std::string schema = get_private_schema_name();
+ std::string user = hawq::test::stringFormat("user%s%d", prefix, idx);
+
+ addUser(&util, prefix, idx, false);
+
+ // all needed policies
+ helper.AddSchemaPolicy("policy10000-1", user, db, schema, {"usage-schema"});
+
+ ret = helper.ActivateAllPoliciesOnRanger();
+ EXPECT_EQ(0,ret);
+ runSQLFile(&util, prefix, "success", idx, false, true, true);
+
+ //delete user
+ delUser(&util, prefix, idx);
+
+ //delete policy
+ helper.DeletePolicyOnRanger("pxfpolicy10000-1");
}
}
@@ -225,6 +217,7 @@ TEST_F(TestHawqRanger, DenyTest) {
runSQLFile(&util, "deny", "succeed", 1);
addPolicy(&util, "deny", 1);
runSQLFile(&util, "deny", "fail", 1);
+ clearEnv(&util, "deny", 1);
}
}
@@ -242,6 +235,8 @@ TEST_F(TestHawqRanger, DenyExcludeTest) {
runSQLFile(&util, "denyexclude", "fail", 2);
addPolicy(&util, "denyexclude", 2);
runSQLFile(&util, "denyexclude", "succeed2", 2);
+ clearEnv(&util, "denyexclude", 2);
+ clearEnv(&util, "deny", 2);
}
}
@@ -259,6 +254,8 @@ TEST_F(TestHawqRanger, AllowExcludeTest) {
addPolicy(&util, "allowexclude", 3);
runSQLFile(&util, "allowexclude", "fail", 3);
+ clearEnv(&util, "allowexclude", 3);
+ clearEnv(&util, "allow", 3);
}
}
@@ -279,6 +276,8 @@ TEST_F(TestHawqRanger, ResourceExcludeTest) {
//add usage-schema to public
addPolicy(&util, "allow", 4);
runSQLFile(&util, "resourceexclude", "succeed", 4);
+ clearEnv(&util, "resourceexclude", 4);
+ clearEnv(&util, "allow", 4);
}
}
@@ -298,6 +297,8 @@ TEST_F(TestHawqRanger, ResourceExcludeStarTest) {
//add usage-schema to public
addPolicy(&util, "allow", 5);
runSQLFile(&util, "resourceexclude", "fail2", 5);
+ clearEnv(&util, "resourceexclude", 5);
+ clearEnv(&util, "allow", 5);
}
}
@@ -311,10 +312,15 @@ TEST_F(TestHawqRanger, ResourceIncludeATest) {
addPolicy(&util, "allow", 6);
runSQLFile(&util, "allow", "fail", 6);
+ clearEnv(&util, "allow", 6);
}
}
-TEST_F(TestHawqRanger, HcatalogTest) {
+
+/**
+ * read HIVE data using hcatalog(read-only) with PXF
+ */
+TEST_F(TestHawqRanger, PXFHcatalogTest) {
SQLUtility util;
if (util.getGUCValue("hawq_acl_type") == "ranger")
{
@@ -322,8 +328,6 @@ TEST_F(TestHawqRanger, HcatalogTest) {
* create a table in hive and populate some rows
*/
clearEnv(&util, "pxf", 1);
- clearEnv(&util, "pxf", 2);
- clearEnv(&util, "pxf", 3);
string rootPath(util.getTestRootPath());
string sqlPath = rootPath + "/Ranger/data/testhive.sql";
auto cmd = hawq::test::stringFormat("hive -f %s", sqlPath.c_str());
@@ -338,13 +342,214 @@ TEST_F(TestHawqRanger, HcatalogTest) {
/*
* add allow policies for this user and query again, succeed.
*/
- addPolicy(&util, "pxf", 1); // usage of default
- addPolicy(&util, "pxf", 2); // select of table
- addPolicy(&util, "pxf", 3); // usage of current schema(e.g.testhawqranger_hcatalogtest)
+ /*
+ usage of default
+ select of table
+ usage of current schema(e.g.testhawqranger_hcatalogtest)
+ */
+ addPolicy(&util, "pxf", 1);
runSQLFile(&util, "pxf", "success", 1);
+
+ //clean
+ clearEnv(&util, "pxf", 1);
+ }
+}
+
+/**
+ * read and write HDFS data using external table with PXF
+ */
+TEST_F(TestHawqRanger, PXFHDFSTest) {
+ SQLUtility util;
+ PolicyHelper helper(util.getTestRootPath(), rangerHost);
+
+ //case idx and folder prefix
+ const char * prefix = "pxf";
+ int idx = 0;
+ int ret = 0;
+
+ if (util.getGUCValue("hawq_acl_type") == "ranger")
+ {
+ std::string db = "hawq_feature_test_db";
+ std::string schema = get_private_schema_name();
+ bool usingDefaultSchema= false;
+ std::string user;
+
+ // clean hdfs folder
+ std::string cmd = "";
+ cmd = hawq::test::stringFormat("hdfs dfs -rm -r /ranger_test/");
+ Command::getCommandStatus(cmd);
+ cmd = hawq::test::stringFormat("hdfs dfs -mkdir /ranger_test/");
+ Command::getCommandStatus(cmd);
+
+ // -- write --
+ idx = 2;
+ addUser(&util, prefix, idx, false);
+ user = hawq::test::stringFormat("user%s%d", prefix ,idx);
+
+ runSQLFile(&util, prefix, "fail", 2, usingDefaultSchema, false, true); // create writable table
+
+ // all needed policies
+ helper.AddSchemaPolicy("pxfpolicy2-1", user, db, schema, {"usage-schema","create"});
+ helper.AddProtocolPolicy("pxfpolicy2-2", user, "pxf", {"insert"});
+ helper.AddTablePolicy("pxfpolicy2-3", user, db, schema, "pxf_hdfs_writabletbl_1", {"insert"});
+
+ ret = helper.ActivateAllPoliciesOnRanger();
+ EXPECT_EQ(0,ret);
+ runSQLFile(&util, prefix, "success", 2, usingDefaultSchema, false, true); // create table
+ runSQLFile(&util, prefix, "success", 3, usingDefaultSchema, false, true); // insert
+
+
+ // -- read --
+ idx = 3;
+ addUser(&util, prefix, idx, false);
+ user = hawq::test::stringFormat("user%s%d", prefix ,idx);
+
+ runSQLFile(&util, prefix, "fail", 4, usingDefaultSchema, false, true); // create readable table
+
+ helper.Reset();
+ // all needed policies
+ helper.AddSchemaPolicy("pxfpolicy3-1", user, db, schema, {"usage-schema","create"});
+ helper.AddProtocolPolicy("pxfpolicy3-2", user, "pxf", {"select"});
+ helper.AddTablePolicy("pxfpolicy3-3", user, db, schema, "pxf_hdfs_textsimple_r1", {"select"});
+
+ ret = helper.ActivateAllPoliciesOnRanger();
+ EXPECT_EQ(0,ret);
+ runSQLFile(&util, prefix, "success", 4, usingDefaultSchema, false, true); // create table
+ runSQLFile(&util, prefix, "success", 5, usingDefaultSchema, false, true); // select
+
+ //delete user
+ delUser(&util, prefix, 2);
+ delUser(&util, prefix, 3);
+
+ //delete policy
+ helper.DeletePolicyOnRanger("pxfpolicy2-1");
+ helper.DeletePolicyOnRanger("pxfpolicy2-2");
+ helper.DeletePolicyOnRanger("pxfpolicy2-3");
+ helper.DeletePolicyOnRanger("pxfpolicy3-1");
+ helper.DeletePolicyOnRanger("pxfpolicy3-2");
+ helper.DeletePolicyOnRanger("pxfpolicy3-3");
}
}
+/**
+ * read Hive data using external table with PXF
+ */
+TEST_F(TestHawqRanger, PXFHiveTest) {
+ SQLUtility util;
+ PolicyHelper helper(util.getTestRootPath(), rangerHost);
+
+ //case idx and folder prefix
+ const char * prefix = "pxf";
+ int idx = 0;
+ int ret = 0;
+
+ if (util.getGUCValue("hawq_acl_type") == "ranger")
+ {
+ std::string db = "hawq_feature_test_db";
+ std::string schema = get_private_schema_name();
+ bool usingDefaultSchema= false;
+ std::string user;
+
+ // create hive table
+ string rootPath(util.getTestRootPath());
+ string sqlPath = rootPath + "/Ranger/data/testhive_externaltable.sql";
+ auto cmd = hawq::test::stringFormat("hive -f %s", sqlPath.c_str());
+ Command::getCommandStatus(cmd);
+
+ // -- read --
+ idx = 4;
+ addUser(&util, prefix, idx, false);
+ user = hawq::test::stringFormat("user%s%d", prefix ,idx);
+
+ runSQLFile(&util, prefix, "fail", 6, usingDefaultSchema, false, true); // create readable table
+
+ helper.Reset();
+ // all needed policies
+ helper.AddSchemaPolicy("pxfpolicy4-1", user, db, schema, {"usage-schema","create"});
+ helper.AddProtocolPolicy("pxfpolicy4-2", user, "pxf", {"select"});
+ helper.AddTablePolicy("pxfpolicy4-3", user, db, schema, "testhive_ext", {"select"});
+
+ ret = helper.ActivateAllPoliciesOnRanger();
+ EXPECT_EQ(0,ret);
+ runSQLFile(&util, prefix, "success", 6, usingDefaultSchema, false, true); // create table
+ runSQLFile(&util, prefix, "success", 7, usingDefaultSchema, false, true); // select
+
+ //delete user
+ delUser(&util, prefix, idx);
+
+ //delete policy
+ helper.DeletePolicyOnRanger("pxfpolicy4-1");
+ helper.DeletePolicyOnRanger("pxfpolicy4-2");
+ helper.DeletePolicyOnRanger("pxfpolicy4-3");
+ }
+}
+
+/**
+ * read HBase data using external table with PXF
+ */
+TEST_F(TestHawqRanger, PXFHBaseTest) {
+ SQLUtility util;
+ PolicyHelper helper(util.getTestRootPath(), rangerHost);
+
+ //case idx and folder prefix
+ const char * prefix = "pxf";
+ int idx = 0;
+ int ret = 0;
+
+ if (util.getGUCValue("hawq_acl_type") == "ranger")
+ {
+ std::string db = "hawq_feature_test_db";
+ std::string schema = get_private_schema_name();
+ bool usingDefaultSchema= false;
+ std::string user;
+
+ // create hbase table
+ auto cmd = hawq::test::stringFormat(
+ "echo \" create 'test_hbase','f1'; put 'test_hbase','r1', 'f1:col1','100' \" | hbase shell");
+ Command::getCommandStatus(cmd);
+
+ // -- read --
+ idx = 5;
+ addUser(&util, prefix, idx, false);
+ user = hawq::test::stringFormat("user%s%d", prefix ,idx);
+
+ runSQLFile(&util, prefix, "fail", 8, usingDefaultSchema, false, true); // create readable table
+
+ helper.Reset();
+ // all needed policies
+ helper.AddSchemaPolicy("pxfpolicy5-1", user, db, schema, {"usage-schema","create"});
+ helper.AddProtocolPolicy("pxfpolicy5-2", user, "pxf", {"select"});
+ helper.AddTablePolicy("pxfpolicy5-3", user, db, schema, "test_hbase", {"select"});
+
+ ret = helper.ActivateAllPoliciesOnRanger();
+ EXPECT_EQ(0,ret);
+ runSQLFile(&util, prefix, "success", 8, usingDefaultSchema, false, true); // create table
+ runSQLFile(&util, prefix, "success", 9, usingDefaultSchema, false, true); // select
+
+ //delete user
+ delUser(&util, prefix, idx);
+
+ //delete policy
+ helper.DeletePolicyOnRanger("pxfpolicy5-1");
+ helper.DeletePolicyOnRanger("pxfpolicy5-2");
+ helper.DeletePolicyOnRanger("pxfpolicy5-3");
+
+ // drop hbase table
+ cmd = hawq::test::stringFormat(
+ "echo \" disable 'test_hbase'; drop 'test_hbase' \" | hbase shell");
+ Command::getCommandStatus(cmd);
+ }
+}
+
+// only drop user in database
+void TestHawqRanger::delUser(hawq::test::SQLUtility* util, std::string case_name, int user_index)
+{
+ string rootPath = util->getTestRootPath();
+ string cmd = "";
+ std::string username = hawq::test::stringFormat("user%s%d", case_name.c_str(), user_index);
+ util->execute(hawq::test::stringFormat("drop role %s;", username.c_str()), false);
+}
+
void TestHawqRanger::addUser(hawq::test::SQLUtility* util, std::string case_name, int user_index, bool full_policy, int writable_index)
{
string rootPath = util->getTestRootPath();
@@ -425,7 +630,7 @@ void TestHawqRanger::clearEnv(hawq::test::SQLUtility* util, std::string case_nam
}
}
-void TestHawqRanger::runSQLFile(hawq::test::SQLUtility* util, std::string case_name, std::string ans_suffix, int sql_index)
+void TestHawqRanger::runSQLFile(hawq::test::SQLUtility* util, std::string case_name, std::string ans_suffix, int sql_index, bool usingDefaultSchema, bool printTupleOnly, bool focus_run)
{
string rootPath = util->getTestRootPath();
auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/%s/*.sql 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), case_name.c_str());
@@ -441,8 +646,8 @@ void TestHawqRanger::runSQLFile(hawq::test::SQLUtility* util, std::string case_n
auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/%spolicy/%d/ 2>/dev/null| grep \"^-\"| wc -l", rootPath.c_str(), case_name.c_str(), i);
int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str());
- if (policy_num > 0){
- util->execSQLFile(deny_sqlfile, deny_ansfile_succeed, initfile);
+ if (focus_run || policy_num > 0){
+ util->execSQLFile(deny_sqlfile, deny_ansfile_succeed, initfile, usingDefaultSchema, printTupleOnly);
}
}
} else {
@@ -452,8 +657,8 @@ void TestHawqRanger::runSQLFile(hawq::test::SQLUtility* util, std::string case_n
auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/%spolicy/%d/ 2>/dev/null| grep \"^-\"| wc -l", rootPath.c_str(), case_name.c_str(), sql_index);
int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str());
- if (policy_num > 0){
- util->execSQLFile(deny_sqlfile, deny_ansfile_succeed, initfile);
+ if (focus_run || policy_num > 0){
+ util->execSQLFile(deny_sqlfile, deny_ansfile_succeed, initfile, usingDefaultSchema, printTupleOnly);
}
}
}
@@ -485,3 +690,15 @@ void TestHawqRanger::addPolicy(hawq::test::SQLUtility* util, std::string case_na
}
sleep(60);
}
+
+/**
+ * get the private schema name based by current test
+ * example: testhawqranger_xxxtest
+ */
+std::string TestHawqRanger::get_private_schema_name()
+{
+ const ::testing::TestInfo *const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
+ string data = string(test_info->test_case_name()) + "_" + test_info->name();
+ std::transform(data.begin(), data.end(), data.begin(), ::tolower);
+ return data;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/Ranger/test_ranger.h
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/test_ranger.h b/src/test/feature/Ranger/test_ranger.h
index 31283f3..d7f78dc 100644
--- a/src/test/feature/Ranger/test_ranger.h
+++ b/src/test/feature/Ranger/test_ranger.h
@@ -33,12 +33,15 @@ public:
std::string initfile;
void clearEnv(hawq::test::SQLUtility* util, std::string case_name, int user_index);
- void runSQLFile(hawq::test::SQLUtility* util, std::string case_name,
- std::string ans_suffix, int sql_index = -1);
+ void runSQLFile(hawq::test::SQLUtility* util, std::string case_name, std::string ans_suffix, int sql_index = -1,
+ bool usingDefaultSchema = false, bool printTupleOnly = false, bool focus_run = false);
void addPolicy(hawq::test::SQLUtility* util, std::string case_name, int policy_index);
void addUser(hawq::test::SQLUtility* util, std::string case_name, int user_index = -1, bool full_policy = false,
int writable_index = -1);
+ void delUser(hawq::test::SQLUtility* util, std::string case_name, int user_index);
+
+ std::string get_private_schema_name();
};
#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/72928dab/src/test/feature/sanity_tests.txt
----------------------------------------------------------------------
diff --git a/src/test/feature/sanity_tests.txt b/src/test/feature/sanity_tests.txt
index bc6df0b..384f6fa 100644
--- a/src/test/feature/sanity_tests.txt
+++ b/src/test/feature/sanity_tests.txt
@@ -2,5 +2,5 @@
#SERIAL=* are the serial tests to run, optional but should not be empty
#you can have several PARALLEL or SRRIAL
-PARALLEL=TestErrorTable.*:TestPreparedStatement.*:TestUDF.*:TestAOSnappy.*:TestAlterOwner.*:TestAlterTable.*:TestCreateTable.*:TestGuc.*:TestType.*:TestDatabase.*:TestParquet.*:TestPartition.*:TestSubplan.*:TestAggregate.*:TestCreateTypeComposite.*:TestGpDistRandom.*:TestInformationSchema.*:TestQueryInsert.*:TestQueryNestedCaseNull.*:TestQueryPolymorphism.*:TestQueryPortal.*:TestQueryPrepare.*:TestQuerySequence.*:TestCommonLib.*:TestToast.*:TestTransaction.*:TestCommand.*:TestCopy.*:TestHawqRegister.TestPartitionTableMultilevel:TestHawqRegister.TestUsage1ExpectSuccessDifferentSchema:TestHawqRegister.TestUsage1ExpectSuccess:TestHawqRegister.TestUsage1SingleHawqFile:TestHawqRegister.TestUsage1SingleHiveFile:TestHawqRegister.TestDataTypes:TestHawqRegister.TestUsage1EofSuccess:TestHawqRegister.TestUsage2Case1Expected:TestHawqRegister.TestUsage2Case2Expected
-SERIAL=TestHawqRanger.*:TestExternalOid.TestExternalOidAll:TestExternalTable.TestExternalTableAll:TestTemp.BasicTest:TestRowTypes.*
+PARALLEL=TestErrorTable.*:TestPreparedStatement.*:TestUDF.*:TestAOSnappy.*:TestAlterOwner.*:TestAlterTable.*:TestCreateTable.*:TestGuc.*:TestType.*:TestDatabase.*:TestParquet.*:TestPartition.*:TestSubplan.*:TestAggregate.*:TestCreateTypeComposite.*:TestGpDistRandom.*:TestInformationSchema.*:TestQueryInsert.*:TestQueryNestedCaseNull.*:TestQueryPolymorphism.*:TestQueryPortal.*:TestQueryPrepare.*:TestQuerySequence.*:TestCommonLib.*:TestToast.*:TestTransaction.*:TestCommand.*:TestCopy.*:TestHawqRegister.TestPartitionTableMultilevel:TestHawqRegister.TestUsage1ExpectSuccessDifferentSchema:TestHawqRegister.TestUsage1ExpectSuccess:TestHawqRegister.TestUsage1SingleHawqFile:TestHawqRegister.TestUsage1SingleHiveFile:TestHawqRegister.TestDataTypes:TestHawqRegister.TestUsage1EofSuccess:TestHawqRegister.TestUsage2Case1Expected:TestHawqRegister.TestUsage2Case2Expected:TestHawqRanger.FallbackTest:TestHawqRanger.PXF*
+SERIAL=TestHawqRanger.BasicTest:TestHawqRanger.Deny*:TestHawqRanger.Allow*:TestHawqRanger.Resource*:TestExternalOid.TestExternalOidAll:TestExternalTable.TestExternalTableAll:TestTemp.BasicTest:TestRowTypes.*