You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sp...@apache.org on 2018/05/29 18:06:53 UTC
[34/43] sentry git commit: SENTRY-2208: Refactor out Sentry service
into own module from sentry-provider-db (Anthony Young-Garner,
reviewed by Sergio Pena, Steve Moist, Na Li)
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java
deleted file mode 100644
index a79ce5f..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/JaasConfiguration.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.service.thrift;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.security.auth.login.AppConfigurationEntry;
-import javax.security.auth.login.Configuration;
-
-/**
- * Creates a programmatic version of a jaas.conf file. This can be used instead of writing a jaas.conf file and setting
- * the system property, "java.security.auth.login.config", to point to that file. It is meant to be used for connecting to
- * ZooKeeper.
- * <p>
- * example usage:
- * JaasConfiguration.addEntry("Client", principal, keytabFile);
- * javax.security.auth.login.Configuration.setConfiguration(JaasConfiguration.getInstance());
- */
-public final class JaasConfiguration extends Configuration {
- private static Map<String, AppConfigurationEntry> entries = new HashMap<String, AppConfigurationEntry>();
- private static JaasConfiguration me = null;
- private static final String krb5LoginModuleName;
-
- static {
- if (System.getProperty("java.vendor").contains("IBM")) {
- krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
- }
- else {
- krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
- }
- }
-
- private JaasConfiguration() {
- // don't need to do anything here but we want to make it private
- }
-
- /**
- * Return the singleton. You'd typically use it only to do this:
- * <p>
- * javax.security.auth.login.Configuration.setConfiguration(JaasConfiguration.getInstance());
- *
- * @return
- */
- public static Configuration getInstance() {
- if (me == null) {
- me = new JaasConfiguration();
- }
- return me;
- }
-
- /**
- * Add an entry to the jaas configuration with the passed in name, principal, and keytab. The other necessary options will be
- * set for you.
- *
- * @param name The name of the entry (e.g. "Client")
- * @param principal The principal of the user
- * @param keytab The location of the keytab
- */
- public static void addEntryForKeytab(String name, String principal, String keytab) {
- Map<String, String> options = new HashMap<String, String>();
- options.put("keyTab", keytab);
- options.put("principal", principal);
- options.put("useKeyTab", "true");
- options.put("storeKey", "true");
- options.put("useTicketCache", "false");
- AppConfigurationEntry entry = new AppConfigurationEntry(krb5LoginModuleName,
- AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options);
- entries.put(name, entry);
- }
-
- /**
- * Add an entry to the jaas configuration with the passed in name. The other
- * necessary options will be set for you.
- *
- * @param name The name of the entry (e.g. "Client")
- */
- public static void addEntryForTicketCache(String sectionName) {
- Map<String, String> options = new HashMap<String, String>();
- options.put("useKeyTab", "false");
- options.put("storeKey", "false");
- options.put("useTicketCache", "true");
- AppConfigurationEntry entry = new AppConfigurationEntry(krb5LoginModuleName,
- AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options);
- entries.put(sectionName, entry);
- }
-
- /**
- * Removes the specified entry.
- *
- * @param name The name of the entry to remove
- */
- public static void removeEntry(String name) {
- entries.remove(name);
- }
-
- /**
- * Clears all entries.
- */
- public static void clearEntries() {
- entries.clear();
- }
-
- /**
- * Returns the entries map.
- *
- * @return the entries map
- */
- public static Map<String, AppConfigurationEntry> getEntries() {
- return entries;
- }
-
- @Override
- public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
- return new AppConfigurationEntry[]{entries.get(name)};
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java
deleted file mode 100644
index 41e4fe4..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/KerberosConfiguration.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.service.thrift;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.security.auth.login.AppConfigurationEntry;
-
-public class KerberosConfiguration extends javax.security.auth.login.Configuration {
- private String principal;
- private String keytab;
- private boolean isInitiator;
- private static final boolean IBM_JAVA = System.getProperty("java.vendor").contains("IBM");
-
- private KerberosConfiguration(String principal, File keytab,
- boolean client) {
- this.principal = principal;
- this.keytab = keytab.getAbsolutePath();
- this.isInitiator = client;
- }
-
- public static javax.security.auth.login.Configuration createClientConfig(String principal,
- File keytab) {
- return new KerberosConfiguration(principal, keytab, true);
- }
-
- public static javax.security.auth.login.Configuration createServerConfig(String principal,
- File keytab) {
- return new KerberosConfiguration(principal, keytab, false);
- }
-
- private static String getKrb5LoginModuleName() {
- return (IBM_JAVA ? "com.ibm.security.auth.module.Krb5LoginModule"
- : "com.sun.security.auth.module.Krb5LoginModule");
- }
-
- @Override
- public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
- Map<String, String> options = new HashMap<String, String>();
-
- if (IBM_JAVA) {
- // IBM JAVA's UseKeytab covers both keyTab and useKeyTab options
- options.put("useKeytab",keytab.startsWith("file://") ? keytab : "file://" + keytab);
-
- options.put("principal", principal);
- options.put("refreshKrb5Config", "true");
-
- // Both "initiator" and "acceptor"
- options.put("credsType", "both");
- } else {
- options.put("keyTab", keytab);
- options.put("principal", principal);
- options.put("useKeyTab", "true");
- options.put("storeKey", "true");
- options.put("doNotPrompt", "true");
- options.put("useTicketCache", "true");
- options.put("renewTGT", "true");
- options.put("refreshKrb5Config", "true");
- options.put("isInitiator", Boolean.toString(isInitiator));
- }
-
- String ticketCache = System.getenv("KRB5CCNAME");
- if (IBM_JAVA) {
- // If cache is specified via env variable, it takes priority
- if (ticketCache != null) {
- // IBM JAVA only respects system property so copy ticket cache to system property
- // The first value searched when "useDefaultCcache" is true.
- System.setProperty("KRB5CCNAME", ticketCache);
- } else {
- ticketCache = System.getProperty("KRB5CCNAME");
- }
-
- if (ticketCache != null) {
- options.put("useDefaultCcache", "true");
- options.put("renewTGT", "true");
- }
- } else {
- if (ticketCache != null) {
- options.put("ticketCache", ticketCache);
- }
- }
- options.put("debug", "true");
-
- return new AppConfigurationEntry[]{
- new AppConfigurationEntry(getKrb5LoginModuleName(),
- AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
- options)};
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java
deleted file mode 100644
index 2a48c63..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ProcessorFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sentry.service.thrift;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.provider.db.service.persistent.SentryStore;
-import org.apache.thrift.TMultiplexedProcessor;
-
-public abstract class ProcessorFactory {
- protected final Configuration conf;
-
- public ProcessorFactory(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * Register a Thrift processor with SentryStore.
- * @param processor a thrift processor.
- * @param sentryStore a {@link SentryStore}
- * @return true if success.
- * @throws Exception
- */
- public abstract boolean register(TMultiplexedProcessor processor,
- SentryStore sentryStore) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
deleted file mode 100644
index b9a0563..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryHMSClient.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
- <p>
- http://www.apache.org/licenses/LICENSE-2.0
- <p>
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Timer;
-import com.codahale.metrics.Timer.Context;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
-import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageDeserializer;
-import org.apache.sentry.provider.db.service.persistent.PathsImage;
-import org.apache.sentry.provider.db.service.persistent.SentryStore;
-import org.apache.sentry.api.service.thrift.SentryMetrics;
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-import static com.codahale.metrics.MetricRegistry.name;
-import static java.util.Collections.emptyMap;
-
-/**
- * Wrapper class for <Code>HiveMetaStoreClient</Code>
- *
- * <p>Abstracts communication with HMS and exposes APi's to connect/disconnect to HMS and to
- * request HMS snapshots and also for new notifications.
- */
-public class SentryHMSClient implements AutoCloseable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SentryHMSClient.class);
- private static final String NOT_CONNECTED_MSG = "Client is not connected to HMS";
-
- private final Configuration conf;
- private HiveMetaStoreClient client = null;
- private HiveConnectionFactory hiveConnectionFactory;
-
- private static final String SNAPSHOT = "snapshot";
- /** Measures time to get full snapshot. */
- private final Timer updateTimer = SentryMetrics.getInstance()
- .getTimer(name(FullUpdateInitializer.class, SNAPSHOT));
- /** Number of times update failed. */
- private final Counter failedSnapshotsCount = SentryMetrics.getInstance()
- .getCounter(name(FullUpdateInitializer.class, "failed"));
-
- public SentryHMSClient(Configuration conf, HiveConnectionFactory hiveConnectionFactory) {
- this.conf = conf;
- this.hiveConnectionFactory = hiveConnectionFactory;
- }
-
- /**
- * Used only for testing purposes.
- *x
- * @param client HiveMetaStoreClient to be initialized
- */
- @VisibleForTesting
- void setClient(HiveMetaStoreClient client) {
- this.client = client;
- }
-
- /**
- * Used to know if the client is connected to HMS
- *
- * @return true if the client is connected to HMS false, if client is not connected.
- */
- boolean isConnected() {
- return client != null;
- }
-
- /**
- * Connects to HMS by creating HiveMetaStoreClient.
- *
- * @throws IOException if could not establish connection
- * @throws InterruptedException if connection was interrupted
- * @throws MetaException if other errors happened
- */
- public void connect()
- throws IOException, InterruptedException, MetaException {
- if (client != null) {
- return;
- }
- client = hiveConnectionFactory.connect().getClient();
- }
-
- /**
- * Disconnects the HMS client.
- */
- public void disconnect() throws Exception {
- try {
- if (client != null) {
- LOGGER.info("Closing the HMS client connection");
- client.close();
- }
- } catch (Exception e) {
- LOGGER.error("failed to close Hive Connection Factory", e);
- } finally {
- client = null;
- }
- }
-
- /**
- * Closes the HMS client.
- *
- * <p>This is similar to disconnect. As this class implements AutoClosable, close should be
- * implemented.
- */
- public void close() throws Exception {
- disconnect();
- }
-
- /**
- * Creates HMS full snapshot.
- *
- * @return Full path snapshot and the last notification id on success
- */
- public PathsImage getFullSnapshot() {
- if (client == null) {
- LOGGER.error(NOT_CONNECTED_MSG);
- return new PathsImage(Collections.<String, Collection<String>>emptyMap(),
- SentryStore.EMPTY_NOTIFICATION_ID, SentryStore.EMPTY_PATHS_SNAPSHOT_ID);
- }
-
- try {
- CurrentNotificationEventId eventIdBefore = client.getCurrentNotificationEventId();
- Map<String, Collection<String>> pathsFullSnapshot = fetchFullUpdate();
- if (pathsFullSnapshot.isEmpty()) {
- LOGGER.info("Received empty paths when getting full snapshot. NotificationID Before Snapshot: {}", eventIdBefore.getEventId());
- return new PathsImage(pathsFullSnapshot, SentryStore.EMPTY_NOTIFICATION_ID,
- SentryStore.EMPTY_PATHS_SNAPSHOT_ID);
- }
-
- CurrentNotificationEventId eventIdAfter = client.getCurrentNotificationEventId();
- LOGGER.info("NotificationID, Before Snapshot: {}, After Snapshot {}",
- eventIdBefore.getEventId(), eventIdAfter.getEventId());
-
- if (eventIdAfter.equals(eventIdBefore)) {
- LOGGER.info("Successfully fetched hive full snapshot, Current NotificationID: {}.",
- eventIdAfter);
- // As eventIDAfter is the last event that was processed, eventIDAfter is used to update
- // lastProcessedNotificationID instead of getting it from persistent store.
- return new PathsImage(pathsFullSnapshot, eventIdAfter.getEventId(),
- SentryStore.EMPTY_PATHS_SNAPSHOT_ID);
- }
-
- LOGGER.info("Reconciling full snapshot - applying {} changes",
- eventIdAfter.getEventId() - eventIdBefore.getEventId());
-
- // While we were taking snapshot, HMS made some changes, so now we need to apply all
- // extra events to the snapshot
- long currentEventId = eventIdBefore.getEventId();
- MessageDeserializer deserializer = new SentryJSONMessageDeserializer();
-
- while (currentEventId < eventIdAfter.getEventId()) {
- NotificationEventResponse response =
- client.getNextNotification(currentEventId, Integer.MAX_VALUE, null);
- if (response == null || !response.isSetEvents() || response.getEvents().isEmpty()) {
- LOGGER.error("Snapshot discarded, updates to HMS data while shapshot is being taken."
- + "ID Before: {}. ID After: {}", eventIdBefore.getEventId(), eventIdAfter.getEventId());
- return new PathsImage(Collections.<String, Collection<String>>emptyMap(),
- SentryStore.EMPTY_NOTIFICATION_ID, SentryStore.EMPTY_PATHS_SNAPSHOT_ID);
- }
-
- for (NotificationEvent event : response.getEvents()) {
- LOGGER.info("Received event = {} currentEventId = {}, eventIdAfter = {}", event.getEventId(), currentEventId, eventIdAfter);
- if (event.getEventId() <= eventIdBefore.getEventId()) {
- LOGGER.error("Received stray event with eventId {} which is less then {}",
- event.getEventId(), eventIdBefore);
- continue;
- }
- if (event.getEventId() > eventIdAfter.getEventId()) {
- // Enough events processed
- LOGGER.debug("Received eventId = {} is greater than eventIdAfter = {}", event.getEventId(), eventIdAfter);
- break;
- }
- try {
- FullUpdateModifier.applyEvent(pathsFullSnapshot, event, deserializer);
- } catch (Exception e) {
- LOGGER.warn("Failed to apply operation", e);
- }
-
- //Log warning message if event id increments are not sequential
- if( event.getEventId() != (currentEventId + 1) ) {
- LOGGER.warn("Received non-sequential event. currentEventId = {} received eventId = {} ", currentEventId, event.getEventId());
- }
- currentEventId = event.getEventId();
- }
- }
-
- LOGGER.info("Successfully fetched hive full snapshot, Current NotificationID: {}.",
- currentEventId);
- // As eventIDAfter is the last event that was processed, eventIDAfter is used to update
- // lastProcessedNotificationID instead of getting it from persistent store.
- return new PathsImage(pathsFullSnapshot, currentEventId,
- SentryStore.EMPTY_PATHS_SNAPSHOT_ID);
- } catch (TException failure) {
- LOGGER.error("Fetching a new HMS snapshot cannot continue because an error occurred during "
- + "the HMS communication: ", failure);
- LOGGER.error("Root Exception", ExceptionUtils.getRootCause(failure));
- return new PathsImage(Collections.<String, Collection<String>>emptyMap(),
- SentryStore.EMPTY_NOTIFICATION_ID, SentryStore.EMPTY_PATHS_SNAPSHOT_ID);
- }
- }
-
- /**
- * Retrieve a Hive full snapshot from HMS.
- *
- * @return HMS snapshot. Snapshot consists of a mapping from auth object name to the set of paths
- * corresponding to that name.
- */
- private Map<String, Collection<String>> fetchFullUpdate() {
- LOGGER.info("Request full HMS snapshot");
- try (FullUpdateInitializer updateInitializer =
- new FullUpdateInitializer(hiveConnectionFactory, conf);
- Context context = updateTimer.time()) {
- Map<String, Collection<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot();
- LOGGER.info("Obtained full HMS snapshot");
- return pathsUpdate;
- } catch (Exception ignored) {
- failedSnapshotsCount.inc();
- LOGGER.error("Snapshot created failed ", ignored);
- return emptyMap();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java
deleted file mode 100644
index efb8ae6..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-import java.io.File;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ThreadFactory;
-
-import javax.security.auth.Subject;
-import javax.security.auth.kerberos.KerberosPrincipal;
-import javax.security.auth.kerberos.KerberosTicket;
-import javax.security.auth.login.LoginContext;
-import javax.security.auth.login.LoginException;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Sets;
-
-public class SentryKerberosContext implements Runnable {
-
- private static final String KERBEROS_RENEWER_THREAD_NAME = "kerberos-renewer-%d";
- private static final float TICKET_RENEW_WINDOW = 0.80f;
- private static final Logger LOGGER = LoggerFactory
- .getLogger(SentryKerberosContext.class);
- private LoginContext loginContext;
- private Subject subject;
- private final javax.security.auth.login.Configuration kerberosConfig;
- private Thread renewerThread;
- private boolean shutDownRenewer = false;
-
- public SentryKerberosContext(String principal, String keyTab, boolean server)
- throws LoginException {
- subject = new Subject(false, Sets.newHashSet(new KerberosPrincipal(principal)),
- new HashSet<Object>(), new HashSet<Object>());
- if(server) {
- kerberosConfig = KerberosConfiguration.createServerConfig(principal, new File(keyTab));
- } else {
- kerberosConfig = KerberosConfiguration.createClientConfig(principal, new File(keyTab));
- }
- loginWithNewContext();
- if (!server) {
- startRenewerThread();
- }
- }
-
- private void loginWithNewContext() throws LoginException {
- LOGGER.info("Logging in with new Context");
- logoutSubject();
- loginContext = new LoginContext("", subject, null, kerberosConfig);
- loginContext.login();
- subject = loginContext.getSubject();
- }
-
- private void logoutSubject() {
- if (loginContext != null) {
- try {
- loginContext.logout();
- } catch (LoginException e) {
- LOGGER.warn("Error logging out the subject", e);
- }
- }
- loginContext = null;
- }
-
- public Subject getSubject() {
- return subject;
- }
-
- /**
- * Get the Kerberos TGT
- * @return the user's TGT or null if none was found
- */
- @Deprecated
- private KerberosTicket getTGT() {
- Set<KerberosTicket> tickets = subject.getPrivateCredentials(KerberosTicket.class);
- for(KerberosTicket ticket: tickets) {
- KerberosPrincipal server = ticket.getServer();
- if (server.getName().equals("krbtgt/" + server.getRealm() +
- "@" + server.getRealm())) {
- return ticket;
- }
- }
- return null;
- }
-
- private long getRefreshTime(KerberosTicket tgt) {
- long start = tgt.getStartTime().getTime();
- long end = tgt.getEndTime().getTime();
- LOGGER.debug("Ticket start time: {}, end time: {}", start, end);
- return start + (long) ((end - start) * TICKET_RENEW_WINDOW);
- }
-
- /***
- * Ticket renewer thread
- * wait till 80% time interval left on the ticket and then renew it
- */
- @Override
- public void run() {
- try {
- LOGGER.info("Sentry Ticket renewer thread started");
- while (!shutDownRenewer) {
- KerberosTicket tgt = getTGT();
- if (tgt == null) {
- LOGGER.warn("No ticket found in the cache");
- return;
- }
- long nextRefresh = getRefreshTime(tgt);
- while (System.currentTimeMillis() < nextRefresh) {
- Thread.sleep(1000);
- if (shutDownRenewer) {
- return;
- }
- }
- loginWithNewContext();
- LOGGER.debug("Renewed ticket");
- }
- } catch (InterruptedException e1) {
- LOGGER.warn("Sentry Ticket renewer thread interrupted", e1);
- return;
- } catch (LoginException e) {
- LOGGER.warn("Failed to renew ticket", e);
- } finally {
- logoutSubject();
- LOGGER.info("Sentry Ticket renewer thread finished");
- }
- }
-
- public void startRenewerThread() {
- ThreadFactory renewerThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat(KERBEROS_RENEWER_THREAD_NAME)
- .build();
- renewerThread = renewerThreadFactory.newThread(this);
- renewerThread.start();
- }
-
- public void shutDown() throws LoginException {
- if (renewerThread != null) {
- shutDownRenewer = true;
- } else {
- logoutSubject();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
deleted file mode 100644
index d92ec21..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java
+++ /dev/null
@@ -1,658 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
-import java.net.ServerSocket;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.EventListener;
-import java.util.List;
-import java.util.concurrent.*;
-
-import javax.security.auth.Subject;
-
-import com.codahale.metrics.Gauge;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SaslRpcServer;
-import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.sentry.Command;
-import org.apache.sentry.api.common.SentryServiceUtil;
-import org.apache.sentry.core.common.utils.SigUtils;
-import org.apache.sentry.provider.db.service.persistent.HMSFollower;
-import org.apache.sentry.provider.db.service.persistent.LeaderStatusMonitor;
-import org.apache.sentry.provider.db.service.persistent.SentryStore;
-import org.apache.sentry.api.service.thrift.SentryHealthCheckServletContextListener;
-import org.apache.sentry.api.service.thrift.SentryMetrics;
-import org.apache.sentry.api.service.thrift.SentryMetricsServletContextListener;
-import org.apache.sentry.api.service.thrift.SentryWebServer;
-import org.apache.sentry.service.common.ServiceConstants;
-import org.apache.sentry.service.common.ServiceConstants.ConfUtilties;
-import org.apache.sentry.service.common.ServiceConstants.ServerConfig;
-import org.apache.thrift.TMultiplexedProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TServerEventHandler;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TSaslServerTransport;
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TServerTransport;
-import org.apache.thrift.transport.TTransportFactory;
-import org.eclipse.jetty.util.MultiException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-import static org.apache.sentry.core.common.utils.SigUtils.registerSigListener;
-
-// Enable signal handler for HA leader/follower status if configured
-public class SentryService implements Callable, SigUtils.SigListener {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SentryService.class);
- private HiveSimpleConnectionFactory hiveConnectionFactory;
-
- private static final String SENTRY_SERVICE_THREAD_NAME = "sentry-service";
- private static final String HMSFOLLOWER_THREAD_NAME = "hms-follower";
- private static final String STORE_CLEANER_THREAD_NAME = "store-cleaner";
- private static final String SERVICE_SHUTDOWN_THREAD_NAME = "service-shutdown";
-
- private enum Status {
- NOT_STARTED,
- STARTED,
- }
-
- private final Configuration conf;
- private final InetSocketAddress address;
- private final int maxThreads;
- private final int minThreads;
- private final boolean kerberos;
- private final String principal;
- private final String[] principalParts;
- private final String keytab;
- private final ExecutorService serviceExecutor;
- private ScheduledExecutorService hmsFollowerExecutor = null;
- private HMSFollower hmsFollower = null;
- private Future serviceStatus;
- private TServer thriftServer;
- private Status status;
- private final int webServerPort;
- private SentryWebServer sentryWebServer;
- private final long maxMessageSize;
- /*
- sentryStore provides the data access for sentry data. It is the singleton instance shared
- between various {@link SentryPolicyService}, i.e., {@link SentryPolicyStoreProcessor} and
- {@link HMSFollower}.
- */
- private final SentryStore sentryStore;
- private ScheduledExecutorService sentryStoreCleanService;
- private final LeaderStatusMonitor leaderMonitor;
-
- public SentryService(Configuration conf) throws Exception {
- this.conf = conf;
- int port = conf
- .getInt(ServerConfig.RPC_PORT, ServerConfig.RPC_PORT_DEFAULT);
- if (port == 0) {
- port = findFreePort();
- conf.setInt(ServerConfig.RPC_PORT, port);
- }
- this.address = NetUtils.createSocketAddr(
- conf.get(ServerConfig.RPC_ADDRESS, ServerConfig.RPC_ADDRESS_DEFAULT),
- port);
- LOGGER.info("Configured on address {}", address);
- kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
- conf.get(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_KERBEROS).trim());
- maxThreads = conf.getInt(ServerConfig.RPC_MAX_THREADS,
- ServerConfig.RPC_MAX_THREADS_DEFAULT);
- minThreads = conf.getInt(ServerConfig.RPC_MIN_THREADS,
- ServerConfig.RPC_MIN_THREADS_DEFAULT);
- maxMessageSize = conf.getLong(ServerConfig.SENTRY_POLICY_SERVER_THRIFT_MAX_MESSAGE_SIZE,
- ServerConfig.SENTRY_POLICY_SERVER_THRIFT_MAX_MESSAGE_SIZE_DEFAULT);
- if (kerberos) {
- // Use Hadoop libraries to translate the _HOST placeholder with actual hostname
- try {
- String rawPrincipal = Preconditions.checkNotNull(conf.get(ServerConfig.PRINCIPAL), ServerConfig.PRINCIPAL + " is required");
- principal = SecurityUtil.getServerPrincipal(rawPrincipal, address.getAddress());
- } catch(IOException io) {
- throw new RuntimeException("Can't translate kerberos principal'", io);
- }
- LOGGER.info("Using kerberos principal: {}", principal);
-
- principalParts = SaslRpcServer.splitKerberosName(principal);
- Preconditions.checkArgument(principalParts.length == 3,
- "Kerberos principal should have 3 parts: " + principal);
- keytab = Preconditions.checkNotNull(conf.get(ServerConfig.KEY_TAB),
- ServerConfig.KEY_TAB + " is required");
- File keytabFile = new File(keytab);
- Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
- "Keytab %s does not exist or is not readable.", keytab);
- } else {
- principal = null;
- principalParts = null;
- keytab = null;
- }
- ThreadFactory sentryServiceThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat(SENTRY_SERVICE_THREAD_NAME)
- .build();
- serviceExecutor = Executors.newSingleThreadExecutor(sentryServiceThreadFactory);
- this.sentryStore = new SentryStore(conf);
- sentryStore.setPersistUpdateDeltas(SentryServiceUtil.isHDFSSyncEnabled(conf));
- this.leaderMonitor = LeaderStatusMonitor.getLeaderStatusMonitor(conf);
- webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT);
-
- status = Status.NOT_STARTED;
-
- // Enable signal handler for HA leader/follower status if configured
- String sigName = conf.get(ServerConfig.SERVER_HA_STANDBY_SIG);
- if ((sigName != null) && !sigName.isEmpty()) {
- LOGGER.info("Registering signal handler {} for HA", sigName);
- try {
- registerSigListener(sigName, this);
- } catch (Exception e) {
- LOGGER.error("Failed to register signal", e);
- }
- }
- }
-
- @Override
- public String call() throws Exception {
- SentryKerberosContext kerberosContext = null;
- try {
- status = Status.STARTED;
- if (kerberos) {
- kerberosContext = new SentryKerberosContext(principal, keytab, true);
- Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- runServer();
- return null;
- }
- });
- } else {
- runServer();
- }
- } catch (Exception t) {
- LOGGER.error("Error starting server", t);
- throw new Exception("Error starting server", t);
- } finally {
- if (kerberosContext != null) {
- kerberosContext.shutDown();
- }
- status = Status.NOT_STARTED;
- }
- return null;
- }
-
- private void runServer() throws Exception {
-
- startSentryStoreCleaner(conf);
- startHMSFollower(conf);
-
- Iterable<String> processorFactories = ConfUtilties.CLASS_SPLITTER
- .split(conf.get(ServerConfig.PROCESSOR_FACTORIES,
- ServerConfig.PROCESSOR_FACTORIES_DEFAULT).trim());
- TMultiplexedProcessor processor = new TMultiplexedProcessor();
- boolean registeredProcessor = false;
- for (String processorFactory : processorFactories) {
- Class<?> clazz = conf.getClassByName(processorFactory);
- if (!ProcessorFactory.class.isAssignableFrom(clazz)) {
- throw new IllegalArgumentException("Processor Factory "
- + processorFactory + " is not a "
- + ProcessorFactory.class.getName());
- }
- try {
- Constructor<?> constructor = clazz
- .getConstructor(Configuration.class);
- LOGGER.info("ProcessorFactory being used: " + clazz.getCanonicalName());
- ProcessorFactory factory = (ProcessorFactory) constructor
- .newInstance(conf);
- boolean registerStatus = factory.register(processor, sentryStore);
- if (!registerStatus) {
- LOGGER.error("Failed to register " + clazz.getCanonicalName());
- }
- registeredProcessor = registerStatus || registeredProcessor;
- } catch (Exception e) {
- throw new IllegalStateException("Could not create "
- + processorFactory, e);
- }
- }
- if (!registeredProcessor) {
- throw new IllegalStateException(
- "Failed to register any processors from " + processorFactories);
- }
- addSentryServiceGauge();
- TServerTransport serverTransport = new TServerSocket(address);
- TTransportFactory transportFactory = null;
- if (kerberos) {
- TSaslServerTransport.Factory saslTransportFactory = new TSaslServerTransport.Factory();
- saslTransportFactory.addServerDefinition(AuthMethod.KERBEROS
- .getMechanismName(), principalParts[0], principalParts[1],
- ServerConfig.SASL_PROPERTIES, new GSSCallback(conf));
- transportFactory = saslTransportFactory;
- } else {
- transportFactory = new TTransportFactory();
- }
- TThreadPoolServer.Args args = new TThreadPoolServer.Args(
- serverTransport).processor(processor)
- .transportFactory(transportFactory)
- .protocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize))
- .minWorkerThreads(minThreads).maxWorkerThreads(maxThreads);
- thriftServer = new TThreadPoolServer(args);
- LOGGER.info("Serving on {}", address);
- startSentryWebServer();
-
- // thriftServer.serve() does not return until thriftServer is stopped. Need to log before
- // calling thriftServer.serve()
- LOGGER.info("Sentry service is ready to serve client requests");
-
- // Allow clients/users watching the console to know when sentry is ready
- System.out.println("Sentry service is ready to serve client requests");
- SentryStateBank.enableState(SentryServiceState.COMPONENT, SentryServiceState.SERVICE_RUNNING);
- thriftServer.serve();
- }
-
- private void startHMSFollower(Configuration conf) throws Exception {
- boolean syncPolicyStore = SentryServiceUtil.isSyncPolicyStoreEnabled(conf);
-
- if ((!SentryServiceUtil.isHDFSSyncEnabled(conf)) && (!syncPolicyStore)) {
- LOGGER.info("HMS follower is not started because HDFS sync is disabled and perm sync is disabled");
- return;
- }
-
- String metastoreURI = SentryServiceUtil.getHiveMetastoreURI();
- if (metastoreURI == null) {
- LOGGER.info("Metastore uri is not configured. Do not start HMSFollower");
- return;
- }
-
- LOGGER.info("Starting HMSFollower to HMS {}", metastoreURI);
-
- Preconditions.checkState(hmsFollower == null);
- Preconditions.checkState(hmsFollowerExecutor == null);
- Preconditions.checkState(hiveConnectionFactory == null);
-
- hiveConnectionFactory = new HiveSimpleConnectionFactory(conf, new HiveConf());
- hiveConnectionFactory.init();
- hmsFollower = new HMSFollower(conf, sentryStore, leaderMonitor, hiveConnectionFactory);
- long initDelay = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS,
- ServerConfig.SENTRY_HMSFOLLOWER_INIT_DELAY_MILLS_DEFAULT);
- long period = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
- ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
- try {
- ThreadFactory hmsFollowerThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat(HMSFOLLOWER_THREAD_NAME)
- .build();
- hmsFollowerExecutor = Executors.newScheduledThreadPool(1, hmsFollowerThreadFactory);
- hmsFollowerExecutor.scheduleAtFixedRate(hmsFollower,
- initDelay, period, TimeUnit.MILLISECONDS);
- } catch (IllegalArgumentException e) {
- LOGGER.error(String.format("Could not start HMSFollower due to illegal argument. period is %s ms",
- period), e);
- throw e;
- }
- }
-
- private void stopHMSFollower(Configuration conf) {
- if ((hmsFollowerExecutor == null) || (hmsFollower == null)) {
- Preconditions.checkState(hmsFollower == null);
- Preconditions.checkState(hmsFollowerExecutor == null);
-
- LOGGER.debug("Skip shuting down hmsFollowerExecutor and closing hmsFollower because they are not created");
- return;
- }
-
- Preconditions.checkNotNull(hmsFollowerExecutor);
- Preconditions.checkNotNull(hmsFollower);
- Preconditions.checkNotNull(hiveConnectionFactory);
-
- // use follower scheduling interval as timeout for shutting down its executor as
- // such scheduling interval should be an upper bound of how long the task normally takes to finish
- long timeoutValue = conf.getLong(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS,
- ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT);
- try {
- SentryServiceUtil.shutdownAndAwaitTermination(hmsFollowerExecutor, "hmsFollowerExecutor",
- timeoutValue, TimeUnit.MILLISECONDS, LOGGER);
- } finally {
- try {
- hiveConnectionFactory.close();
- } catch (Exception e) {
- LOGGER.error("Can't close HiveConnectionFactory", e);
- }
- hmsFollowerExecutor = null;
- hiveConnectionFactory = null;
- try {
- // close connections
- hmsFollower.close();
- } catch (Exception ex) {
- LOGGER.error("HMSFollower.close() failed", ex);
- } finally {
- hmsFollower = null;
- }
- }
- }
-
- private void startSentryStoreCleaner(Configuration conf) {
- Preconditions.checkState(sentryStoreCleanService == null);
-
- // If SENTRY_STORE_CLEAN_PERIOD_SECONDS is set to positive, the background SentryStore cleaning
- // thread is enabled. Currently, it only purges the delta changes {@link MSentryChange} in
- // the sentry store.
- long storeCleanPeriodSecs = conf.getLong(
- ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS,
- ServerConfig.SENTRY_STORE_CLEAN_PERIOD_SECONDS_DEFAULT);
- if (storeCleanPeriodSecs <= 0) {
- return;
- }
-
- try {
- Runnable storeCleaner = new Runnable() {
- @Override
- public void run() {
- if (leaderMonitor.isLeader()) {
- sentryStore.purgeDeltaChangeTables();
- sentryStore.purgeNotificationIdTable();
- }
- }
- };
-
- ThreadFactory sentryStoreCleanerThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat(STORE_CLEANER_THREAD_NAME)
- .build();
- sentryStoreCleanService = Executors.newSingleThreadScheduledExecutor(sentryStoreCleanerThreadFactory);
- sentryStoreCleanService.scheduleWithFixedDelay(
- storeCleaner, 0, storeCleanPeriodSecs, TimeUnit.SECONDS);
-
- LOGGER.info("sentry store cleaner is scheduled with interval {} seconds", storeCleanPeriodSecs);
- }
- catch(IllegalArgumentException e){
- LOGGER.error("Could not start SentryStoreCleaner due to illegal argument", e);
- sentryStoreCleanService = null;
- }
- }
-
- private void stopSentryStoreCleaner() {
- Preconditions.checkNotNull(sentryStoreCleanService);
-
- try {
- SentryServiceUtil.shutdownAndAwaitTermination(sentryStoreCleanService, "sentryStoreCleanService",
- 10, TimeUnit.SECONDS, LOGGER);
- }
- finally {
- sentryStoreCleanService = null;
- }
- }
-
- private void addSentryServiceGauge() {
- SentryMetrics.getInstance().addSentryServiceGauges(this);
- }
-
- private void startSentryWebServer() throws Exception{
- Boolean sentryReportingEnable = conf.getBoolean(ServerConfig.SENTRY_WEB_ENABLE,
- ServerConfig.SENTRY_WEB_ENABLE_DEFAULT);
- if(sentryReportingEnable) {
- List<EventListener> listenerList = new ArrayList<>();
- listenerList.add(new SentryHealthCheckServletContextListener());
- listenerList.add(new SentryMetricsServletContextListener());
- sentryWebServer = new SentryWebServer(listenerList, webServerPort, conf);
- sentryWebServer.start();
- }
- }
-
- private void stopSentryWebServer() throws Exception{
- if( sentryWebServer != null) {
- sentryWebServer.stop();
- sentryWebServer = null;
- }
- }
-
- public InetSocketAddress getAddress() {
- return address;
- }
-
- public synchronized boolean isRunning() {
- return status == Status.STARTED && thriftServer != null
- && thriftServer.isServing();
- }
-
- public synchronized void start() throws Exception{
- if (status != Status.NOT_STARTED) {
- throw new IllegalStateException("Cannot start when " + status);
- }
- LOGGER.info("Attempting to start...");
- serviceStatus = serviceExecutor.submit(this);
- }
-
- public synchronized void stop() throws Exception{
- MultiException exception = null;
- LOGGER.info("Attempting to stop...");
- leaderMonitor.close();
- if (isRunning()) {
- LOGGER.info("Attempting to stop sentry thrift service...");
- try {
- thriftServer.stop();
- thriftServer = null;
- status = Status.NOT_STARTED;
- } catch (Exception e) {
- LOGGER.error("Error while stopping sentry thrift service", e);
- exception = addMultiException(exception,e);
- }
- } else {
- thriftServer = null;
- status = Status.NOT_STARTED;
- LOGGER.info("Sentry thrift service is already stopped...");
- }
- if (isWebServerRunning()) {
- try {
- LOGGER.info("Attempting to stop sentry web service...");
- stopSentryWebServer();
- } catch (Exception e) {
- LOGGER.error("Error while stopping sentry web service", e);
- exception = addMultiException(exception,e);
- }
- } else {
- LOGGER.info("Sentry web service is already stopped...");
- }
-
- stopHMSFollower(conf);
- stopSentryStoreCleaner();
-
- if (exception != null) {
- exception.ifExceptionThrow();
- }
- SentryStateBank.disableState(SentryServiceState.COMPONENT,SentryServiceState.SERVICE_RUNNING);
- LOGGER.info("Stopped...");
- }
-
- /**
- * If the current daemon is active, make it standby.
- * Here 'active' means it is the only daemon that can fetch snapshots from HMA and write
- * to the backend DB.
- */
- @VisibleForTesting
- public synchronized void becomeStandby() {
- leaderMonitor.deactivate();
- }
-
- private MultiException addMultiException(MultiException exception, Exception e) {
- MultiException newException = exception;
- if (newException == null) {
- newException = new MultiException();
- }
- newException.add(e);
- return newException;
- }
-
- private boolean isWebServerRunning() {
- return sentryWebServer != null
- && sentryWebServer.isAlive();
- }
-
- private static int findFreePort() {
- int attempts = 0;
- while (attempts++ <= 1000) {
- try {
- ServerSocket s = new ServerSocket(0);
- int port = s.getLocalPort();
- s.close();
- return port;
- } catch (IOException e) {
- // ignore and retry
- }
- }
- throw new IllegalStateException("Unable to find a port after 1000 attempts");
- }
-
- public static Configuration loadConfig(String configFileName)
- throws MalformedURLException {
- File configFile = null;
- if (configFileName == null) {
- throw new IllegalArgumentException("Usage: "
- + ServiceConstants.ServiceArgs.CONFIG_FILE_LONG
- + " path/to/sentry-service.xml");
- } else if (!((configFile = new File(configFileName)).isFile() && configFile
- .canRead())) {
- throw new IllegalArgumentException("Cannot read configuration file "
- + configFile);
- }
- Configuration conf = new Configuration(false);
- conf.addResource(configFile.toURI().toURL(), true);
- return conf;
- }
-
- public static class CommandImpl implements Command {
- @Override
- public void run(String[] args) throws Exception {
- CommandLineParser parser = new GnuParser();
- Options options = new Options();
- options.addOption(ServiceConstants.ServiceArgs.CONFIG_FILE_SHORT,
- ServiceConstants.ServiceArgs.CONFIG_FILE_LONG,
- true, "Sentry Service configuration file");
- CommandLine commandLine = parser.parse(options, args);
- String configFileName = commandLine.getOptionValue(ServiceConstants.
- ServiceArgs.CONFIG_FILE_LONG);
- File configFile = null;
- if (configFileName == null || commandLine.hasOption("h") || commandLine.hasOption("help")) {
- // print usage
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("sentry --command service", options);
- System.exit(-1);
- } else if(!((configFile = new File(configFileName)).isFile() && configFile.canRead())) {
- throw new IllegalArgumentException("Cannot read configuration file " + configFile);
- }
- Configuration serverConf = loadConfig(configFileName);
- final SentryService server = new SentryService(serverConf);
- server.start();
-
- ThreadFactory serviceShutdownThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat(SERVICE_SHUTDOWN_THREAD_NAME)
- .build();
- Runtime.getRuntime().addShutdownHook(serviceShutdownThreadFactory.newThread(new Runnable() {
- @Override
- public void run() {
- LOGGER.info("ShutdownHook shutting down server");
- try {
- server.stop();
- } catch (Throwable t) {
- LOGGER.error("Error stopping SentryService", t);
- System.exit(1);
- }
- }
- }));
-
- // Let's wait on the service to stop
- try {
- // Wait for the service thread to finish
- server.serviceStatus.get();
- } finally {
- server.serviceExecutor.shutdown();
- }
- }
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- /**
- * Add Thrift event handler to underlying thrift threadpool server
- * @param eventHandler
- */
- public void setThriftEventHandler(TServerEventHandler eventHandler) throws IllegalStateException {
- if (thriftServer == null) {
- throw new IllegalStateException("Server is not initialized or stopped");
- }
- thriftServer.setServerEventHandler(eventHandler);
- }
-
- public TServerEventHandler getThriftEventHandler() throws IllegalStateException {
- if (thriftServer == null) {
- throw new IllegalStateException("Server is not initialized or stopped");
- }
- return thriftServer.getEventHandler();
- }
-
- public Gauge<Boolean> getIsActiveGauge() {
- return new Gauge<Boolean>() {
- @Override
- public Boolean getValue() {
- return leaderMonitor.isLeader();
- }
- };
- }
-
- public Gauge<Long> getBecomeActiveCount() {
- return new Gauge<Long>() {
- @Override
- public Long getValue() {
- return leaderMonitor.getLeaderCount();
- }
- };
- }
-
- @Override
- public void onSignal(String signalName) {
- // Become follower
- leaderMonitor.deactivate();
- }
-
- /**
- * Restart HMSFollower with new configuration
- * @param newConf Configuration
- * @throws Exception
- */
- @VisibleForTesting
- public void restartHMSFollower(Configuration newConf) throws Exception{
- stopHMSFollower(conf);
- startHMSFollower(newConf);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
deleted file mode 100644
index d146a0d..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceClientFactory.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sentry.core.common.transport.RetryClientInvocationHandler;
-import org.apache.sentry.core.common.transport.SentryPolicyClientTransportConfig;
-import org.apache.sentry.core.common.transport.SentryTransportFactory;
-import org.apache.sentry.core.common.transport.SentryTransportPool;
-import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient;
-import org.apache.sentry.api.service.thrift.SentryPolicyServiceClientDefaultImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.ThreadSafe;
-import java.lang.reflect.Proxy;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * Client factory for Hive clients. The factory uses connection pooling.
- */
-@ThreadSafe
-public final class SentryServiceClientFactory {
- private static final Logger LOGGER = LoggerFactory.getLogger(SentryServiceClientFactory.class);
-
- private static final SentryPolicyClientTransportConfig transportConfig =
- new SentryPolicyClientTransportConfig();
- private final Configuration conf;
- private final SentryTransportPool transportPool;
-
- /** Keep track of singleton instances */
- private static final AtomicReference<SentryServiceClientFactory> clientFactory =
- new AtomicReference<>();
-
- /**
- * Create a client instance. The supplied configuration is only used the first time and
- * ignored afterwords. Tests that want to supply different configurations
- * should call {@link #factoryReset(SentryServiceClientFactory)} to force new configuration
- * read.
- * @param conf Configuration
- * @return client instance
- * @throws Exception
- */
- public static SentryPolicyServiceClient create(Configuration conf) throws Exception {
- SentryServiceClientFactory factory = clientFactory.get();
- if (factory != null) {
- return factory.create();
- }
- factory = new SentryServiceClientFactory(conf);
- boolean ok = clientFactory.compareAndSet(null, factory);
- if (ok) {
- return factory.create();
- }
- // Close old factory
- factory.close();
- return clientFactory.get().create();
- }
-
- /**
- * Create a new instance of the factory which will hand hand off connections from
- * the pool.
- * @param conf Configuration object
- */
- private SentryServiceClientFactory(Configuration conf) {
- this.conf = conf;
-
- transportPool = new SentryTransportPool(conf, transportConfig,
- new SentryTransportFactory(conf, transportConfig));
- }
-
- private SentryPolicyServiceClient create() throws Exception {
- return (SentryPolicyServiceClient) Proxy
- .newProxyInstance(SentryPolicyServiceClientDefaultImpl.class.getClassLoader(),
- SentryPolicyServiceClientDefaultImpl.class.getInterfaces(),
- new RetryClientInvocationHandler(conf,
- new SentryPolicyServiceClientDefaultImpl(conf, transportPool), transportConfig));
- }
-
- /**
- * Reset existing factory and return the old one.
- * Only used by tests.
- */
- public static SentryServiceClientFactory factoryReset(SentryServiceClientFactory factory) {
- LOGGER.debug("factory reset");
- return clientFactory.getAndSet(factory);
- }
-
- public void close() {
- try {
- transportPool.close();
- } catch (Exception e) {
- LOGGER.error("failed to close transport pool", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java
deleted file mode 100644
index c1d81ed..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceFactory.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class SentryServiceFactory {
- public static SentryService create(Configuration conf) throws Exception {
- return new SentryService(conf);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java
deleted file mode 100644
index 4219adc..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceState.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with the License. You may obtain
- * a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.sentry.service.thrift;
-
-/**
- * States for the SentryService
- */
-public enum SentryServiceState implements SentryState {
- /**
- * The SentryService is running all of its threads and services. This include the store cleaner,
- * the web interface, the HMS poller, and the Thrift Server
- */
- SERVICE_RUNNING,
-
- /**
- * A full update of data from the HMS is running by the thread handling the update.
- */
- FULL_UPDATE_RUNNING;
-
- /**
- * The component name this state is for.
- */
- public static final String COMPONENT = "SentryService";
-
- /**
- * {@inheritDoc}
- */
- @Override
- public long getValue() {
- return 1 << this.ordinal();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java
deleted file mode 100644
index 040d82a..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryState.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with the License. You may obtain
- * a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.sentry.service.thrift;
-
-/**
- * Interface for SentryState enums.
- */
-public interface SentryState {
-
- /**
- * This gets the Bitmask value associated with the state.
- */
- long getValue();
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java
deleted file mode 100644
index 2c05d49..0000000
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryStateBank.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with the License. You may obtain
- * a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.sentry.service.thrift;
-
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.AtomicLongMap;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import javax.annotation.concurrent.ThreadSafe;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * <p>SentryStateBank is a state visibility manager to allow components to communicate state to other
- * parts of the application.</p>
- *
- * <p>It allows you to provide multiple boolean states for a component and expose those states to
- * other parts of the application without having references to the actual instances of the classes
- * setting those states.</p>
- *
- * <p>SentryStateBank uses a bitmasked long in order to store the states, so its very compact and
- * efficient.</p>
- *
- * <p>States are defined using an enum that implements the {@link SentryState} interface. The
- * {@link SentryState} implementation can provide up to 64 states per components. The {@link SentryState#getValue()}
- * implementation should return a bitshift of the oridinal of the enum value. This gives the bitmask
- * location to be checking for the state.</p>
- *
- * <p>The following is an example of a simple {@link SentryState} enum implementation</p>
- *
- * <pre>
- * {@code
- *
- * public enum ExampleState implements SentryState {
- * FIRST_STATE,
- * SECOND_STATE;
- *
- * public static final String COMPONENT = "ExampleState";
- *
- * @Override
- * public long getValue() {
- * return 1 << this.ordinal();
- * }
- * }
- * }
- * </pre>
- *
- * <p>This class is thread safe. It uses a {@link ReentrantReadWriteLock} to wrap accesses and changes
- * to the state.</p>
- */
-@ThreadSafe
-public final class SentryStateBank {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SentryStateBank.class);
- private static final AtomicLongMap<String> states = AtomicLongMap.create();
- private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-
- protected SentryStateBank() {
- }
-
- @VisibleForTesting
- static void clearAllStates() {
- states.clear();
- LOGGER.debug("All states have been cleared.");
- }
-
- @VisibleForTesting
- static void resetComponentState(String component) {
- states.remove(component);
- LOGGER.debug("All states have been cleared for component {}", component);
- }
-
- /**
- * Enables a state
- *
- * @param component the component for the state
- * @param state the state to disable
- */
- public static void enableState(String component, SentryState state) {
- lock.writeLock().lock();
- try {
- states.put(component, states.get(component) | state.getValue());
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("{} entered state {}", component, state.toString());
- }
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Disables a state for a component
- *
- * @param component the component for the state
- * @param state the state to disable
- */
- public static void disableState(String component, SentryState state) {
- lock.writeLock().lock();
- try {
- states.put(component, states.get(component) & (~state.getValue()));
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("{} exited state {}", component, state.toString());
- }
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Returns if a state is enabled or not
- *
- * @param component The component for the state
- * @param state the SentryState to check
- * @return true if the state for the component is enabled
- */
- public static boolean isEnabled(String component, SentryState state) {
- lock.readLock().lock();
- try {
- return (states.get(component) & state.getValue()) == state.getValue();
- } finally {
- lock.readLock().unlock();
- }
-
- }
-
- /**
- * Checks if all of the states passed in are enabled
- *
- * @param component The component for the states
- * @param passedStates the SentryStates to check
- */
- public static boolean hasStatesEnabled(String component, Set<SentryState> passedStates) {
- lock.readLock().lock();
- try {
- long value = 0L;
-
- for (SentryState state : passedStates) {
- value += state.getValue();
- }
- return (states.get(component) & value) == value;
- } finally {
- lock.readLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sentry/blob/b97f5c7a/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html b/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html
deleted file mode 100644
index 9f52a8e..0000000
--- a/sentry-provider/sentry-provider-db/src/main/webapp/SentryService.html
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<!DOCTYPE HTML>
-<html lang="en">
- <head>
- <meta charset="utf-8">
- <title>Sentry Service</title>
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <meta name="description" content="">
- <link href="css/bootstrap.min.css" rel="stylesheet">
- <link href="css/bootstrap-theme.min.css" rel="stylesheet">
- <link href="css/sentry.css" rel="stylesheet">
- </head>
-
- <body>
- <nav class="navbar navbar-default navbar-fixed-top">
- <div class="container">
- <div class="navbar-header">
- <a class="navbar-brand" href="#"><img src="sentry.png" alt="Sentry Logo"/></a>
- </div>
- <div class="collapse navbar-collapse">
- <ul class="nav navbar-nav">
- <li class="active"><a href="#">Home</a></li>
- <li><a href="/metrics?pretty=true">Metrics</a></li>
- <li><a href="/threads">Threads</a></li>
- <li><a href="/conf">Configuration</a></li>
- </ul>
- </div>
- </div>
- </nav>
-
- <div class="container">
- <div class="page-header"><h2>Sentry Service</h2></div>
- <ul>
- <li><a href="/metrics?pretty=true">Metrics</a></li>
- <li><a href="/threads">Threads</a></li>
- <li><a href="/conf">Configuration</a></li>
- </ul>
- </div>
-
- <footer class="footer">
- <div class="container">
- <p class="text-muted">SENTRY 2.0.0-SNAPSHOT</p>
- </div>
- </footer>
- </body>
-</html>