You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sr...@apache.org on 2016/08/16 00:26:09 UTC
[2/2] sentry git commit: Sentry-1321: Implement HMSFollower in Sentry
service which reads the NotificationLog entries (Sravya Tirukkovalur,
Reviewed by: Hao Hao) (Append)
Sentry-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao) (Append)
Change-Id: If6e3d4a754b71cc0ba0cec1e34a44e6d9816ea38
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/908072d6
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/908072d6
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/908072d6
Branch: refs/heads/sentry-ha-redesign
Commit: 908072d66bbabd91491a7d8fe8d9fa558dbc9c46
Parents: 2d5ed99
Author: Sravya Tirukkovalur <sr...@apache.org>
Authored: Mon Aug 15 17:25:47 2016 -0700
Committer: Sravya Tirukkovalur <sr...@apache.org>
Committed: Mon Aug 15 17:25:47 2016 -0700
----------------------------------------------------------------------
sentry-binding/sentry-binding-hive-conf/pom.xml | 62 +++
.../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ++++++++++++
.../sentry-binding-hive-follower/pom.xml | 71 ++++
.../json/JSONAlterPartitionMessage.java | 78 ++++
.../messaging/json/JSONAlterTableMessage.java | 68 +++
.../json/SentryJSONAddPartitionMessage.java | 49 +++
.../json/SentryJSONAlterPartitionMessage.java | 53 +++
.../json/SentryJSONAlterTableMessage.java | 50 +++
.../json/SentryJSONCreateDatabaseMessage.java | 44 ++
.../json/SentryJSONCreateTableMessage.java | 45 ++
.../json/SentryJSONDropDatabaseMessage.java | 44 ++
.../json/SentryJSONDropPartitionMessage.java | 49 +++
.../json/SentryJSONDropTableMessage.java | 45 ++
.../json/SentryJSONMessageDeserializer.java | 110 +++++
.../json/SentryJSONMessageFactory.java | 177 ++++++++
.../SentryInvalidHMSEventException.java | 31 ++
.../exception/SentryStandbyException.java | 4 +
.../SentryPolicyServiceClientDefaultImpl.java | 12 +-
.../sentry/service/thrift/HMSFollower.java | 411 +++++++++++++++++++
.../thrift/PoolClientInvocationHandler.java | 305 +++++++++++---
.../thrift/SentryServiceClientFactory.java | 11 +-
.../thrift/SentryServiceClientPoolFactory.java | 11 +-
.../thrift/TestSentryServiceFailureCase.java | 5 +-
.../TestSentryServiceWithInvalidMsgSize.java | 2 +-
.../sentry/service/thrift/TestHMSFollower.java | 165 ++++++++
.../thrift/TestPoolClientInvocationHandler.java | 69 ++++
26 files changed, 2173 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-conf/pom.xml b/sentry-binding/sentry-binding-hive-conf/pom.xml
new file mode 100644
index 0000000..2cae541
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-conf/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-binding</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>sentry-binding-hive-conf</artifactId>
+ <name>Sentry Hive Binding Configuration</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
new file mode 100644
index 0000000..4de755f
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.hive.conf;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HiveAuthzConf extends Configuration {
+
+ /**
+ * Configuration key used in hive-site.xml to point at sentry-site.xml
+ */
+ public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url";
+ public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url";
+ public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name";
+ public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name";
+ public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors";
+ public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation";
+ public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error";
+ public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges";
+ /**
+ * Property used to persist the role set in the session. This is not public for now.
+ */
+ public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set";
+
+ public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST =
+ "hive.sentry.security.command.whitelist";
+ public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT =
+ "set,reset,reload";
+
+ public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist";
+ public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2";
+
+ // Disable the serde Uri privileges by default for backward compatibilities.
+ public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges";
+ public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false;
+
+ public static final String HIVE_UDF_WHITE_LIST =
+ "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," +
+ "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," +
+ "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," +
+ "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," +
+ "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" +
+ ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," +
+ "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," +
+ "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," +
+ "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," +
+ "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," +
+ "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," +
+ "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," +
+ "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," +
+ "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," +
+ "context_ngrams,ewah_bitmap,compute_stats,percentile," +
+ "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," +
+ "index,in_file,instr,locate,elt,concat_ws,sort_array," +
+ "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," +
+ "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," +
+ "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," +
+ "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," +
+ "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath";
+
+ public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method";
+
+ /**
+ * Config setting definitions
+ */
+ public static enum AuthzConfVars {
+ AUTHZ_PROVIDER("sentry.provider",
+ "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"),
+ AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""),
+ AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"),
+ AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"),
+ AUTHZ_POLICY_FILE_FORMATTER(
+ "sentry.hive.policy.file.formatter",
+ "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"),
+ AUTHZ_SERVER_NAME("sentry.hive.server", ""),
+ AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"),
+ SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"),
+ AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"),
+ AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""),
+ AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null),
+ AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"),
+ AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"),
+ AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"),
+
+ AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider",
+ "org.apache.sentry.provider.file.ResourceAuthorizationProvider"),
+ AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""),
+ AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""),
+ AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"),
+ SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"),
+ AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"),
+ AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", "");
+
+ private final String varName;
+ private final String defaultVal;
+
+ AuthzConfVars(String varName, String defaultVal) {
+ this.varName = varName;
+ this.defaultVal = defaultVal;
+ }
+
+ public String getVar() {
+ return varName;
+ }
+
+ public String getDefault() {
+ return defaultVal;
+ }
+
+ public static String getDefault(String varName) {
+ for (AuthzConfVars oneVar : AuthzConfVars.values()) {
+ if(oneVar.getVar().equalsIgnoreCase(varName)) {
+ return oneVar.getDefault();
+ }
+ }
+ return null;
+ }
+ }
+
+ // map of current property names - > deprecated property names.
+ // The binding layer code should work if the deprecated property names are provided,
+ // as long as the new property names aren't also provided. Since the binding code
+ // only calls the new property names, we require a map from current names to deprecated
+ // names in order to check if the deprecated name of a property was set.
+ private static final Map<String, AuthzConfVars> currentToDeprecatedProps =
+ new HashMap<String, AuthzConfVars>();
+ static {
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED);
+ currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED);
+ };
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HiveAuthzConf.class);
+ public static final String AUTHZ_SITE_FILE = "sentry-site.xml";
+ private final String hiveAuthzSiteFile;
+
+ public HiveAuthzConf(URL hiveAuthzSiteURL) {
+ super();
+ LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
+ addResource(hiveAuthzSiteURL);
+ applySystemProperties();
+ LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
+ this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString();
+ }
+ /**
+ * Apply system properties to this object if the property name is defined in ConfVars
+ * and the value is non-null and not an empty string.
+ */
+ private void applySystemProperties() {
+ Map<String, String> systemProperties = getConfSystemProperties();
+ for (Entry<String, String> systemProperty : systemProperties.entrySet()) {
+ this.set(systemProperty.getKey(), systemProperty.getValue());
+ }
+ }
+
+ /**
+ * This method returns a mapping from config variable name to its value for all config variables
+ * which have been set using System properties
+ */
+ public static Map<String, String> getConfSystemProperties() {
+ Map<String, String> systemProperties = new HashMap<String, String>();
+
+ for (AuthzConfVars oneVar : AuthzConfVars.values()) {
+ String value = System.getProperty(oneVar.getVar());
+ if (value != null && value.length() > 0) {
+ systemProperties.put(oneVar.getVar(), value);
+ }
+ }
+ return systemProperties;
+ }
+
+ @Override
+ public String get(String varName) {
+ return get(varName, null);
+ }
+
+ @Override
+ public String get(String varName, String defaultVal) {
+ String retVal = super.get(varName);
+ if (retVal == null) {
+ // check if the deprecated value is set here
+ if (currentToDeprecatedProps.containsKey(varName)) {
+ retVal = super.get(currentToDeprecatedProps.get(varName).getVar());
+ }
+ if (retVal == null) {
+ retVal = AuthzConfVars.getDefault(varName);
+ } else {
+ LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() +
+ " instead of " + varName);
+ }
+ }
+ if (retVal == null) {
+ retVal = defaultVal;
+ }
+ return retVal;
+ }
+
+ public String getHiveAuthzSiteFile() {
+ return hiveAuthzSiteFile;
+ }
+
+ /**
+ * Extract the authz config file path from given hive conf and load the authz config
+ * @param hiveConf
+ * @return
+ * @throws IllegalArgumentException
+ */
+ public static HiveAuthzConf getAuthzConf(HiveConf hiveConf)
+ throws IllegalArgumentException {
+ boolean depreicatedConfigFile = false;
+
+ String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL);
+ if (hiveAuthzConf == null
+ || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) {
+ hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL);
+ depreicatedConfigFile = true;
+ }
+
+ if (hiveAuthzConf == null
+ || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) {
+ throw new IllegalArgumentException("Configuration key "
+ + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf
+ + "' is invalid.");
+ }
+
+ try {
+ return new HiveAuthzConf(new URL(hiveAuthzConf));
+ } catch (MalformedURLException e) {
+ if (depreicatedConfigFile) {
+ throw new IllegalArgumentException("Configuration key "
+ + HiveAuthzConf.HIVE_ACCESS_CONF_URL
+ + " specifies a malformed URL '" + hiveAuthzConf + "'", e);
+ } else {
+ throw new IllegalArgumentException("Configuration key "
+ + HiveAuthzConf.HIVE_SENTRY_CONF_URL
+ + " specifies a malformed URL '" + hiveAuthzConf + "'", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/pom.xml b/sentry-binding/sentry-binding-hive-follower/pom.xml
new file mode 100644
index 0000000..bcfb417
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-binding</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>sentry-binding-hive-follower</artifactId>
+ <name>Hive follower for Sentry</name>
+
+ <properties>
+ <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
+ <datanucleus-core.version>3.2.12</datanucleus-core.version>
+ <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-core</artifactId>
+ <version>${datanucleus-core.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-api-jdo</artifactId>
+ <version>${datanucleus-api-jdo.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-rdbms</artifactId>
+ <version>${datanucleus-rdbms.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-server-extensions</artifactId>
+ <version>${hive.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
new file mode 100644
index 0000000..890186b
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+/*
+* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor
+ */
+public class JSONAlterPartitionMessage extends AlterPartitionMessage {
+ @JsonProperty
+ String server;
+ @JsonProperty
+ String servicePrincipal;
+ @JsonProperty
+ String db;
+ @JsonProperty
+ String table;
+ @JsonProperty
+ Long timestamp;
+ @JsonProperty
+ List<String> values;
+
+ public JSONAlterPartitionMessage() {}
+ public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.db = db;
+ this.table = table;
+ this.timestamp = timestamp;
+ this.values = values;
+ this.checkValid();
+ }
+
+ public String getServer() {
+ return this.server;
+ }
+
+ public String getServicePrincipal() {
+ return this.servicePrincipal;
+ }
+
+ public String getDB() {
+ return this.db;
+ }
+
+ public Long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+
+ public List<String> getValues() {
+ return this.values;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
new file mode 100644
index 0000000..76211c3
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * This class is required as this class does not have a default contructor in Hive 1.1.0
+ */
+public class JSONAlterTableMessage extends AlterTableMessage {
+ @JsonProperty
+ String server;
+ @JsonProperty
+ String servicePrincipal;
+ @JsonProperty
+ String db;
+ @JsonProperty
+ String table;
+ @JsonProperty
+ Long timestamp;
+
+ public JSONAlterTableMessage() {}
+ public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) {
+ this.server = server;
+ this.servicePrincipal = servicePrincipal;
+ this.db = db;
+ this.table = table;
+ this.timestamp = timestamp;
+ this.checkValid();
+ }
+
+ public String getServer() {
+ return this.server;
+ }
+
+ public String getServicePrincipal() {
+ return this.servicePrincipal;
+ }
+
+ public String getDB() {
+ return this.db;
+ }
+
+ public Long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
new file mode 100644
index 0000000..c0c469c
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage {
+ @JsonProperty
+ List<String> locations;
+
+ public SentryJSONAddPartitionMessage() {
+ }
+
+ public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<Map<String, String>> partitions, Long timestamp, List<String> locations) {
+ super(server, servicePrincipal, db, table, partitions, timestamp);
+ this.locations = locations;
+ }
+
+ public List<String> getLocations() {
+ return locations;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
new file mode 100644
index 0000000..99eb67a
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{
+ @JsonProperty
+ String location;
+ @JsonProperty
+ String oldLocation;
+
+ public SentryJSONAlterPartitionMessage() {
+ }
+
+ public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<String> values, Long timestamp, String oldlocation, String newLocation) {
+ super(server, servicePrincipal, db, table, values, timestamp);
+ this.location = newLocation;
+ this.oldLocation = oldlocation;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ public String getOldLocation() {
+ return oldLocation;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
new file mode 100644
index 0000000..6e59e25
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONAlterTableMessage extends JSONAlterTableMessage {
+ @JsonProperty
+ String location; //newLocation
+ @JsonProperty
+ String oldLocation;
+
+ public SentryJSONAlterTableMessage() {
+ }
+
+ public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table,
+ Long timestamp, String oldLocation, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ this.oldLocation = oldLocation;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+ public String getOldLocation() {
+ return oldLocation;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
new file mode 100644
index 0000000..ba19cbe
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONCreateDatabaseMessage() {
+ }
+
+ public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+ super(server, servicePrincipal, db, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
new file mode 100644
index 0000000..57d11d2
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateTableMessage extends JSONCreateTableMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONCreateTableMessage() {
+ }
+
+ public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
new file mode 100644
index 0000000..05f83f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropDatabaseMessage() {
+ }
+
+ public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+ super(server, servicePrincipal, db, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
new file mode 100644
index 0000000..2ab61f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropPartitionMessage() {
+ }
+
+ public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table,
+ List<Map<String, String>> partitions, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, partitions, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
new file mode 100644
index 0000000..7005776
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+
+public class SentryJSONDropTableMessage extends JSONDropTableMessage {
+ @JsonProperty
+ String location;
+
+ public SentryJSONDropTableMessage() {
+ }
+
+ public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+ super(server, servicePrincipal, db, table, timestamp);
+ this.location = location;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+
+ @Override
+ public String toString() {
+ return SentryJSONMessageDeserializer.serialize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
new file mode 100644
index 0000000..b645c45
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.*;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+public class SentryJSONMessageDeserializer extends MessageDeserializer {
+ static ObjectMapper mapper = new ObjectMapper();
+
+ public SentryJSONMessageDeserializer() {
+ }
+
+ /**
+ * Method to de-serialize CreateDatabaseMessage instance.
+ */
+ public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) {
+ try {
+ return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) {
+ try {
+ return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3);
+ }
+ }
+
+ public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) {
+ try {
+ return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) {
+ try {
+ return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropTableMessage getDropTableMessage(String messageBody) {
+ try {
+ return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3);
+ }
+ }
+
+ public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3);
+ }
+ }
+
+ public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3);
+ }
+ }
+
+ public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) {
+ try {
+ return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class);
+ } catch (Exception var3) {
+ throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3);
+ }
+ }
+
+ static {
+ mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
+ public static String serialize(Object object) {
+ try {
+ return mapper.writeValueAsString(object);
+ }
+ catch (Exception exception) {
+ throw new IllegalArgumentException("Could not serialize: ", exception);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
new file mode 100644
index 0000000..00e7db8
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hive.hcatalog.messaging.*;
+
+import java.util.*;
+
+public class SentryJSONMessageFactory extends MessageFactory {
+ private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName());
+ private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+ public SentryJSONMessageFactory() {
+ LOG.info("Using SentryJSONMessageFactory for building Notification log messages ");
+
+ }
+ public MessageDeserializer getDeserializer() {
+ return deserializer;
+ }
+
+ public String getVersion() {
+ return "0.1";
+ }
+
+ public String getMessageFormat() {
+ return "json";
+ }
+
+ public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) {
+ return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+ Long.valueOf(this.now()), db.getLocationUri());
+ }
+ public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) {
+ return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+ Long.valueOf(this.now()), db.getLocationUri());
+ }
+
+ public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) {
+ return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+ }
+
+ public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) {
+ return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation());
+ }
+
+ public SentryJSONDropTableMessage buildDropTableMessage(Table table) {
+ return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+ }
+
+ public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) {
+ return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()),
+ getPartitionLocations(partitions));
+ }
+
+ private List<String> getPartitionLocations(List<Partition> partitions) {
+ List<String> paths = new ArrayList<String>();
+ for(Partition partition:partitions) {
+ paths.add(partition.getSd().getLocation());
+ }
+ return paths;
+ }
+
+ //TODO: Not sure what is this used for. Need to investigate
+ private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) {
+ Iterator<Partition> iterator = partitionSpec.getPartitionIterator();
+ List<String> locations = new ArrayList<String>();
+ while(iterator.hasNext()) {
+ locations.add(iterator.next().getSd().getLocation());
+ }
+ return locations;
+ }
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) {
+ return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+ table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()),
+ getPartitionLocations(partitionSpec));
+ }
+
+ public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) {
+ /*
+ f (partitionEvent.getOldPartition() != null) {
+ oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
+ }
+ if (partitionEvent.getNewPartition() != null) {
+ newLoc = partitionEvent.getNewPartition().getSd().getLocation();
+ }
+
+ if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) {
+ String authzObj =
+ partitionEvent.getOldPartition().getDbName() + "."
+ + partitionEvent.getOldPartition().getTableName();
+ for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+ plugin.renameAuthzObject(authzObj, oldLoc,
+ authzObj, newLoc);
+ }
+ }
+ * */
+ return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+ before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(),
+ after.getSd().getLocation());
+ }
+
+ public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) {
+ return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
+ partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
+ Long.valueOf(this.now()), partition.getSd().getLocation());
+ }
+
+ private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) {
+ LinkedHashMap partitionKeys = new LinkedHashMap();
+
+ for(int i = 0; i < table.getPartitionKeysSize(); ++i) {
+ partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i));
+ }
+
+ return partitionKeys;
+ }
+
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) {
+ ArrayList partitionList = new ArrayList(partitions.size());
+ Iterator i$ = partitions.iterator();
+
+ while(i$.hasNext()) {
+ Partition partition = (Partition)i$.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+
+ return partitionList;
+ }
+
+ @InterfaceAudience.LimitedPrivate({"Hive"})
+ @InterfaceStability.Evolving
+ private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) {
+ ArrayList partitionList = new ArrayList();
+ PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator();
+
+ while(iterator.hasNext()) {
+ Partition partition = (Partition)iterator.next();
+ partitionList.add(getPartitionKeyValues(table, partition));
+ }
+
+ return partitionList;
+ }
+ //This is private in parent class
+ private long now() {
+ return System.currentTimeMillis() / 1000L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
new file mode 100644
index 0000000..50fb59c
--- /dev/null
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.core.common.exception;
+
+public class SentryInvalidHMSEventException extends SentryUserException {
+ private static final long serialVersionUID = 29620806553835L;
+ public SentryInvalidHMSEventException(String msg) {
+ super(msg);
+ }
+ public SentryInvalidHMSEventException(String msg, String reason) {
+ super(msg, reason);
+ }
+ public SentryInvalidHMSEventException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
index b2df699..da6cfce 100644
--- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
@@ -34,4 +34,8 @@ public class SentryStandbyException extends SentryUserException {
public SentryStandbyException(String msg, String reason) {
super(msg, reason);
}
+
+ public SentryStandbyException(String msg, Throwable t) {
+ super(msg, t);
+ }
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
index 1039e6e..4f42a51 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
@@ -132,12 +132,18 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
}
public SentryPolicyServiceClientDefaultImpl(Configuration conf) throws IOException {
+ this(Preconditions.checkNotNull(conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key "
+ + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt(
+ ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT), conf);
+ }
+
+ public SentryPolicyServiceClientDefaultImpl(String addr, int port,
+ Configuration conf) throws IOException {
this.conf = conf;
Preconditions.checkNotNull(this.conf, "Configuration object cannot be null");
this.serverAddress = NetUtils.createSocketAddr(Preconditions.checkNotNull(
- conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key "
- + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt(
- ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT));
+ addr, "Config key " + ClientConfig.SERVER_RPC_ADDRESS
+ + " is required"), port);
this.connectionTimeout = conf.getInt(ClientConfig.SERVER_RPC_CONN_TIMEOUT,
ClientConfig.SERVER_RPC_CONN_TIMEOUT_DEFAULT);
kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
new file mode 100644
index 0000000..4430471
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -0,0 +1,411 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
+import org.apache.sentry.core.common.exception.*;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.sentry.binding.metastore.messaging.json.*;
+
+import javax.security.auth.Subject;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
+import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
+
+/*
+HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
+It gets the full update and notification logs from HMS and applies it to
+update permissions stored in Sentry using SentryStore and also update the <obj,path> state
+stored for HDFS- Sentry sync.
+ */
+@SuppressWarnings("PMD")
+public class HMSFollower implements Runnable {
+ private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class);
+
+ private long currentEventID;
+ private static boolean connectedToHMS = false;
+ private HiveMetaStoreClient client;
+ private SentryKerberosContext kerberosContext;
+ private Configuration authzConf;
+ private boolean kerberos;
+ private SentryStore sentryStore;
+ private String hiveInstance;
+ final static int maxRetriesForLogin = 3;
+ final static int maxRetriesForConnection = 3;
+
+ HMSFollower(Configuration conf) throws SentryNoSuchObjectException,
+ SentryAccessDeniedException, SentrySiteConfigurationException, IOException { //TODO: Handle any possible exceptions or throw specific exceptions
+ LOGGER.info("HMSFollower is being initialized");
+ authzConf = conf;
+ try {
+ sentryStore = new SentryStore(authzConf);
+ } catch (SentryStandbyException e) {
+ //TODO: Do not start HMSFollower if standby
+ }
+ //TODO: Initialize currentEventID from Sentry db
+ currentEventID = 0;
+ }
+
+ @VisibleForTesting
+ HMSFollower(Configuration conf, SentryStore sentryStore, String hiveInstance) {
+ this.authzConf = conf;
+ this.sentryStore = sentryStore;
+ this.hiveInstance = hiveInstance;
+ }
+
+ @VisibleForTesting
+ public static boolean isConnectedToHMS() {
+ return connectedToHMS;
+ }
+
+ /*
+ Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
+ Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
+ Throws @MetaException if there was a problem on creating an HMSClient
+ */
+ private HiveMetaStoreClient getMetaStoreClient(Configuration conf)
+ throws LoginException, MetaException {
+ if(client != null) {
+ return client;
+ }
+ // Seems like HMS client creation although seems successful,
+ // it actually connects to an invalid HMS instance.
+ // So it seems like it is necessary to wait until we make sure metastore config is properly loaded.
+ boolean loadedHiveConf = HiveConf.isLoadMetastoreConfig();
+ if(!loadedHiveConf) {
+ return null;
+ }
+ final HiveConf hiveConf = new HiveConf();
+ hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar());
+
+ String principal, keytab;
+
+ //TODO: Is this the right(standard) way to create a HMS client? HiveMetastoreClientFactoryImpl?
+ //TODO: Check if HMS is using kerberos instead of relying on Sentry conf
+ //TODO: Handle TGT renewals
+ kerberos = ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
+ conf.get(ServiceConstants.ServerConfig.SECURITY_MODE, ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS).trim());
+ if (kerberos) {
+ LOGGER.info("Making a kerberos connection to HMS");
+ //TODO: Is this needed? Use Hadoop libraries to translate the _HOST placeholder with actual hostname
+ //Validate principal
+ principal = Preconditions.checkNotNull(ServiceConstants.ServerConfig.PRINCIPAL,
+ ServiceConstants.ServerConfig.PRINCIPAL + " is required");
+ LOGGER.info("Using kerberos principal: " + principal);
+ final String[] principalParts = SaslRpcServer.splitKerberosName(principal);
+ Preconditions.checkArgument(principalParts.length == 3,
+ "Kerberos principal should have 3 parts: " + principal);
+
+ keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB),
+ ServiceConstants.ServerConfig.KEY_TAB + " is required");
+ File keytabFile = new File(keytab);
+ Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
+ "Keytab " + keytab + " does not exist or is not readable.");
+ boolean establishedKerberosContext = false;
+ int attempt = 1;
+ while(establishedKerberosContext) {
+ try {
+ kerberosContext = new SentryKerberosContext(principal, keytab, true);
+ establishedKerberosContext = true;
+ LOGGER.info("Established kerberos context, will now connect to HMS");
+ } catch (LoginException e) {
+ //Kerberos login failed
+ if( attempt > maxRetriesForLogin ) {
+ throw e;
+ }
+ attempt++;
+ }
+ }
+ boolean establishedConnection = false;
+ attempt = 1;
+ while(establishedConnection) {
+ try {
+ client = Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<HiveMetaStoreClient>() {
+ @Override
+ public HiveMetaStoreClient run() throws Exception {
+ return new HiveMetaStoreClient(hiveConf);
+ }
+ });
+ LOGGER.info("Secure connection established with HMS");
+ } catch (PrivilegedActionException e) {
+ if( attempt > maxRetriesForConnection ) {
+ //We should just retry as it is possible that HMS is not ready yet to receive requests
+ //TODO: How do we differentiate between kerberos problem versus HMS not being up?
+ LOGGER.error("Cannot connect to HMS", e);
+ }
+ attempt++;
+ }
+ }
+ } else {
+ //This is only for testing purposes. Sentry strongly recommends strong authentication
+ client = new HiveMetaStoreClient(hiveConf);
+ LOGGER.info("Non secure connection established with HMS");
+ }
+ return client;
+ }
+
+ public void run() {
+ if( client == null ) {
+ try {
+ client = getMetaStoreClient(authzConf);
+ if (client == null) {
+ //TODO: Do we want to throw an exception after a certain timeout?
+ return;
+ } else {
+ connectedToHMS = true;
+ LOGGER.info("HMSFollower of Sentry successfully connected to HMS");
+ }
+ } catch (Exception e) {
+
+ }
+ }
+ if (needFullUpdate()) {
+ //TODO: Handle
+ } else {
+ try {
+ NotificationEventResponse response = client.getNextNotification(currentEventID, Integer.MAX_VALUE, null);
+ if (response.isSetEvents()) {
+ LOGGER.info(String.format("CurrentEventID = %s. Processing %s events",
+ currentEventID, response.getEvents().size()));
+ processNotificationEvents(response.getEvents());
+ }
+ } catch (TException e) {
+ LOGGER.error("ThriftException occured fetching Notification entries, will try");
+ e.printStackTrace();
+ } catch (SentryInvalidInputException|SentryInvalidHMSEventException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private boolean needFullUpdate() {
+ //TODO Implement
+ return false;
+ }
+
+ private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) {
+ return "true"
+ .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true")));
+ }
+
+ /*
+ Throws SentryInvalidHMSEventException if Notification event contains insufficient information
+ */
+
+ void processNotificationEvents(List<NotificationEvent> events) throws
+ SentryInvalidHMSEventException, SentryInvalidInputException {
+ SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+
+ for (NotificationEvent event : events) {
+ String dbName, tableName, oldLocation, newLocation, location;
+ switch (HCatEventMessage.EventType.valueOf(event.getEventType())) {
+ case CREATE_DATABASE:
+ SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage());
+ dbName = message.getDB();
+
+ location = message.getLocation();
+ if (dbName == null || location == null) {
+ throw new SentryInvalidHMSEventException(String.format("Create database event has incomplete information. " +
+ "dbName = %s location = %s", dbName, location));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+ try {
+ dropSentryDbPrivileges(dbName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.addPath(dbName, location)
+ break;
+ case DROP_DATABASE:
+ SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage());
+ dbName = dropDatabaseMessage.getDB();
+ if (dbName == null) {
+ throw new SentryInvalidHMSEventException(String.format("Drop database event has incomplete information. " +
+ "dbName = %s", dbName));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+ try {
+ dropSentryDbPrivileges(dbName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Drop database event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.deletePath(dbName, location)
+ break;
+ case CREATE_TABLE:
+ SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(event.getMessage());
+ dbName = createTableMessage.getDB();
+ tableName = createTableMessage.getTable();
+ location = createTableMessage.getLocation();
+ if (dbName == null || tableName == null || location == null) {
+ throw new SentryInvalidHMSEventException(String.format("Create table event has incomplete information. " +
+ "dbName = %s, tableName = %s, location = %s", dbName, tableName, location));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+ try {
+ dropSentryTablePrivileges(dbName, tableName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.deletePath(dbName, location)
+ break;
+ case DROP_TABLE:
+ SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage());
+ dbName = dropTableMessage.getDB();
+ tableName = dropTableMessage.getTable();
+ if (dbName == null || tableName == null) {
+ throw new SentryInvalidHMSEventException(String.format("Drop table event has incomplete information. " +
+ "dbName = %s, tableName = %s", dbName, tableName));
+ }
+ if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+ try{
+ dropSentryTablePrivileges(dbName, tableName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Drop table event. Event: " + event.toString(), e);
+ }
+ }
+ //TODO: HDFSPlugin.deletePath(dbName, location)
+ break;
+ case ALTER_TABLE:
+ SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(event.getMessage());
+
+ String oldDbName = alterTableMessage.getDB();
+ String oldTableName = alterTableMessage.getTable();
+ String newDbName = event.getDbName();
+ String newTableName = event.getTableName();
+ oldLocation = alterTableMessage.getOldLocation();
+ newLocation = alterTableMessage.getLocation();
+
+ if (oldDbName == null || oldTableName == null || newDbName == null || newTableName == null ||
+ oldLocation == null || newLocation == null) {
+ throw new SentryInvalidHMSEventException(String.format("Alter table event has incomplete information. " +
+ "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s",
+ oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation));
+ }
+
+ if(!newDbName.equalsIgnoreCase(oldDbName) || !oldTableName.equalsIgnoreCase(newTableName)) { // Name has changed
+ if(!oldLocation.equals(newLocation)) { // Location has changed
+
+ //Name and path has changed
+ // - Alter table rename for managed table
+ //TODO: Handle HDFS plugin
+
+ } else {
+ //Only name has changed
+ // - Alter table rename for an external table
+ //TODO: Handle HDFS plugin
+
+ }
+ try {
+ renamePrivileges(oldDbName, oldTableName, newDbName, newTableName);
+ } catch (SentryNoSuchObjectException e) {
+ LOGGER.info("Rename Sentry privilege ignored as there are no privileges on the table: %s.%s", oldDbName, oldTableName);
+ } catch (SentryInvalidInputException e) {
+ throw new SentryInvalidInputException("Could not process Alter table event. Event: " + event.toString(), e);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ } else if(!oldLocation.equals(newLocation)) { // Only Location has changed{
+ //- Alter table set location
+ //TODO: Handle HDFS plugin
+ } else {
+ LOGGER.info(String.format("Alter table notification ignored as neither name nor location has changed: " +
+ "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s",
+ oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation));
+ }
+ //TODO: Write test cases for all these cases
+ break;
+ case ADD_PARTITION:
+ case DROP_PARTITION:
+ case ALTER_PARTITION:
+ //TODO: Handle HDFS plugin
+ break;
+ }
+ currentEventID = event.getEventId();
+ }
+ }
+
+ private void dropSentryDbPrivileges(String dbName) throws SentryNoSuchObjectException, SentryInvalidInputException {
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setDb(dbName);
+ try {
+ sentryStore.dropPrivilege(authorizable);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ }
+ private void dropSentryTablePrivileges(String dbName, String tableName) throws SentryNoSuchObjectException,
+ SentryInvalidInputException {
+ TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+ authorizable.setDb(dbName);
+ authorizable.setTable(tableName);
+ try {
+ sentryStore.dropPrivilege(authorizable);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ }
+ private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws
+ SentryNoSuchObjectException, SentryInvalidInputException, SentryStandbyException{
+ TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance);
+ oldAuthorizable.setDb(oldDbName);
+ oldAuthorizable.setTable(oldTableName);
+ TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+ newAuthorizable.setDb(newDbName);
+ newAuthorizable.setTable(newTableName);
+ try {
+ sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable);
+ } catch (SentryStandbyException e) {
+ LOGGER.error("Seems like this process became a standby. " +
+ "Ignoring the failure. The new leader will reprocess this notification event");
+ }
+ }
+}