You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2018/10/16 20:53:15 UTC
hive git commit: HIVE-20731: keystore file in JdbcStorageHandler
should be authorized (Daniel Dai, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master d7be4b9f2 -> dc8d8e134
HIVE-20731: keystore file in JdbcStorageHandler should be authorized (Daniel Dai, reviewed by Thejas Nair)
Signed-off-by: Thejas M Nair <th...@hortonworks.com>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dc8d8e13
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dc8d8e13
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dc8d8e13
Branch: refs/heads/master
Commit: dc8d8e134c2cc752e89d1b6ccf3097c8e43aa88a
Parents: d7be4b9
Author: Daniel Dai <da...@gmail.com>
Authored: Tue Oct 16 13:52:05 2018 -0700
Committer: Daniel Dai <da...@gmail.com>
Committed: Tue Oct 16 13:52:13 2018 -0700
----------------------------------------------------------------------
.../hive/ql/parse/BaseSemanticAnalyzer.java | 27 ++++++++
.../hive/ql/parse/DDLSemanticAnalyzer.java | 1 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 1 +
.../authorization_jdbc_keystore.q | 28 +++++++++
.../queries/clientpositive/external_jdbc_auth.q | 26 ++++++++
.../authorization_jdbc_keystore.q.out | 1 +
.../llap/external_jdbc_auth.q.out | 66 +++++++++++++++++++-
7 files changed, 148 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index 1df5c74..c9df668 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -98,6 +99,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2275,4 +2279,27 @@ public abstract class BaseSemanticAnalyzer {
public WriteEntity getAcidAnalyzeTable() {
return null;
}
+
+ public void addPropertyReadEntry(Map<String, String> tblProps, Set<ReadEntity> inputs) throws SemanticException {
+ if (tblProps.containsKey(Constants.JDBC_KEYSTORE)) {
+ try {
+ String keystore = tblProps.get(Constants.JDBC_KEYSTORE);
+ Configuration conf = new Configuration();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, keystore);
+ boolean found = false;
+ for (CredentialProvider provider : CredentialProviderFactory.getProviders(conf))
+ if (provider instanceof AbstractJavaKeyStoreProvider) {
+ Path path = ((AbstractJavaKeyStoreProvider) provider).getPath();
+ inputs.add(toReadEntity(path));
+ found = true;
+ }
+ if (!found) {
+ throw new SemanticException("Cannot recognize keystore " + keystore + ", only JavaKeyStoreProvider is " +
+ "supported");
+ }
+ } catch (IOException e) {
+ throw new SemanticException(e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 29f6ecf..bba7d6c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1771,6 +1771,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
alterTblDesc.setDropIfExists(true);
}
} else {
+ addPropertyReadEntry(mapProp, inputs);
alterTblDesc = new AlterTableDesc(AlterTableTypes.ADDPROPS, partSpec, expectView);
}
alterTblDesc.setProps(mapProp);
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 6a6e6c3..eed875e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13297,6 +13297,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
break;
case HiveParser.TOK_TABLEPROPERTIES:
tblProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0));
+ addPropertyReadEntry(tblProps, inputs);
break;
case HiveParser.TOK_TABLESERIALIZER:
child = (ASTNode) child.getChild(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q b/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q
new file mode 100644
index 0000000..63288f7
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/authorization_jdbc_keystore.q
@@ -0,0 +1,28 @@
+--! qt:dataset:
+
+set hive.test.authz.sstd.hs2.mode=true;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator;
+set hive.security.authorization.enabled=true;
+
+dfs -cp ${system:test.tmp.dir}/../../../../data/files/test.jceks ${system:test.tmp.dir}/test.jceks;
+dfs -chmod 555 ${system:test.tmp.dir}/test.jceks;
+
+CREATE EXTERNAL TABLE ext_auth1
+(
+ ikey int,
+ bkey bigint,
+ fkey float,
+ dkey double
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "DERBY",
+ "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver",
+ "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;collation=TERRITORY_BASED:PRIMARY",
+ "hive.sql.dbcp.username" = "user1",
+ "hive.sql.dbcp.password.keystore" = "jceks://file/${system:test.tmp.dir}/test.jceks",
+ "hive.sql.dbcp.password.key" = "test_derby_auth1.password",
+ "hive.sql.table" = "SIMPLE_DERBY_TABLE1",
+ "hive.sql.dbcp.maxActive" = "1"
+);
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/test/queries/clientpositive/external_jdbc_auth.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/external_jdbc_auth.q b/ql/src/test/queries/clientpositive/external_jdbc_auth.q
index acfb298..f4cbe94 100644
--- a/ql/src/test/queries/clientpositive/external_jdbc_auth.q
+++ b/ql/src/test/queries/clientpositive/external_jdbc_auth.q
@@ -9,6 +9,18 @@ SELECT
dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;create=true','user1','passwd1',
'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ),
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ),
+
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ),
+
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user1\', \'passwd1\')' ),
+
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ),
+
dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1',
'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
@@ -30,6 +42,18 @@ SELECT
dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2;create=true','user2','passwd2',
'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ),
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ),
+
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ),
+
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user2\', \'passwd2\')' ),
+
+dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2',
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ),
+
dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2',
'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
@@ -92,3 +116,5 @@ CREATE TABLE hive_table
INSERT INTO hive_table VALUES(20);
(SELECT * FROM ext_auth1 JOIN hive_table ON ext_auth1.ikey=hive_table.ikey) UNION ALL (SELECT * FROM ext_auth2 JOIN hive_table ON ext_auth2.ikey=hive_table.ikey);
+
+ALTER TABLE ext_auth1 SET TBLPROPERTIES ("hive.sql.dbcp.password.keystore" = "jceks://file/${system:test.tmp.dir}/../../../data/files/test.jceks");
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out b/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out
new file mode 100644
index 0000000..0b8182a
--- /dev/null
+++ b/ql/src/test/results/clientnegative/authorization_jdbc_keystore.q.out
@@ -0,0 +1 @@
+#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/dc8d8e13/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out
index badc8b9..b299a38 100644
--- a/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out
+++ b/ql/src/test/results/clientpositive/llap/external_jdbc_auth.q.out
@@ -12,6 +12,18 @@ SELECT
'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ),
#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user1\', \'passwd1\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ),
+
+#### A masked pattern was here ####
'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
#### A masked pattern was here ####
@@ -35,6 +47,18 @@ SELECT
'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ),
#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user1\', \'passwd1\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ),
+
+#### A masked pattern was here ####
'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
#### A masked pattern was here ####
@@ -50,7 +74,7 @@ limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 0 0 0 0
+0 0 0 0 0 0 0 0 0
PREHOOK: query: FROM src
SELECT
@@ -59,6 +83,18 @@ SELECT
'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ),
#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user2\', \'passwd2\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ),
+
+#### A masked pattern was here ####
'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
#### A masked pattern was here ####
@@ -82,6 +118,18 @@ SELECT
'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ),
#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.connection.requireAuthentication\', \'true\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.authentication.provider\', \'BUILTIN\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.user.user2\', \'passwd2\')' ),
+
+#### A masked pattern was here ####
+'CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(\'derby.database.propertiesOnly\', \'true\')' ),
+
+#### A masked pattern was here ####
'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'),
#### A masked pattern was here ####
@@ -97,7 +145,7 @@ limit 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 0 0 0 0
+0 0 0 0 0 0 0 0 0
PREHOOK: query: CREATE EXTERNAL TABLE ext_auth1
(
ikey int,
@@ -117,6 +165,7 @@ TBLPROPERTIES (
"hive.sql.dbcp.maxActive" = "1"
)
PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
PREHOOK: Output: database:default
PREHOOK: Output: default@ext_auth1
POSTHOOK: query: CREATE EXTERNAL TABLE ext_auth1
@@ -138,6 +187,7 @@ TBLPROPERTIES (
"hive.sql.dbcp.maxActive" = "1"
)
POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ext_auth1
PREHOOK: query: CREATE EXTERNAL TABLE ext_auth2
@@ -159,6 +209,7 @@ TBLPROPERTIES (
"hive.sql.dbcp.maxActive" = "1"
)
PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
PREHOOK: Output: database:default
PREHOOK: Output: default@ext_auth2
POSTHOOK: query: CREATE EXTERNAL TABLE ext_auth2
@@ -180,6 +231,7 @@ TBLPROPERTIES (
"hive.sql.dbcp.maxActive" = "1"
)
POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ext_auth2
PREHOOK: query: CREATE TABLE hive_table
@@ -219,3 +271,13 @@ POSTHOOK: Input: default@hive_table
#### A masked pattern was here ####
20 20 20.0 20.0 20
20 20 20.0 20.0 20
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@ext_auth1
+#### A masked pattern was here ####
+PREHOOK: Output: default@ext_auth1
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@ext_auth1
+#### A masked pattern was here ####
+POSTHOOK: Output: default@ext_auth1