You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by co...@apache.org on 2018/04/30 16:35:41 UTC
[5/9] sentry git commit: SENTRY-2207 Refactor out Sentry CLI from
sentry-provider-db into own module. Steve Moist,
reviewed by Colm O hEigeartaigh.
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini b/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini
deleted file mode 100644
index c1bfe4b..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-corporal = corporal_role
-sergeant = corporal_role, sergeant_role
-general = corporal_role, sergeant_role, general_role
-commander_in_chief = corporal_role, sergeant_role, general_role, commander_in_chief_role
-
-[roles]
-corporal_role = indexer=info->action=read, \
- indexer=info->action=write
-sergeant_role = indexer=info->action=write
-general_role = indexer=info->action=*
-commander_in_chief_role = indexer=*
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini b/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini
deleted file mode 100644
index 03083a7..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-
-[roles]
-
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini
deleted file mode 100644
index fbbebfc..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-groupa = RoLe1
-groupb = rOlE1
-groupc = ROLE2
-
-[roles]
-RoLe1 = collection=*
-rOlE1 = collection=*
-ROLE2 = collection=*
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini
deleted file mode 100644
index da7df4c..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-corporal = corporal_role
-sergeant = corporal_role, sergeant_role
-general = corporal_role, sergeant_role, general_role
-commander_in_chief = corporal_role, sergeant_role, general_role, commander_in_chief_role
-
-[roles]
-corporal_role = collection=info->action=query, \
- collection=info->action=update
-sergeant_role = collection=info->action=update
-general_role = collection=info->action=*
-commander_in_chief_role = collection=*
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini
deleted file mode 100644
index 03083a7..0000000
--- a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[groups]
-
-[roles]
-
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-tools/pom.xml b/sentry-tools/pom.xml
index 4d8fc89..b882c6f 100644
--- a/sentry-tools/pom.xml
+++ b/sentry-tools/pom.xml
@@ -26,6 +26,7 @@ limitations under the License.
<modelVersion>4.0.0</modelVersion>
<artifactId>sentry-tools</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
<dependencies>
<dependency>
@@ -33,27 +34,56 @@ limitations under the License.
<artifactId>sentry-binding-hive</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-provider-db</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-provider-db</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.budhash.cliche</groupId>
<artifactId>cliche-shell</artifactId>
<version>0.9.3</version>
</dependency>
<dependency>
- <groupId>org.apache.sentry</groupId>
- <artifactId>sentry-provider-db</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
- <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <reuseForks>false</reuseForks>
+ </configuration>
+ </plugin>
+ </plugins>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
</build>
-
-
</project>
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java b/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
index e92155c..495d4cd 100644
--- a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
+++ b/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java
@@ -24,7 +24,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.log4j.PropertyConfigurator;
import org.apache.sentry.binding.hive.authz.SentryConfigTool;
-import org.apache.sentry.provider.db.tools.SentrySchemaTool;
+import org.apache.sentry.cli.tools.SentrySchemaTool;
import org.apache.sentry.service.thrift.SentryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java
new file mode 100644
index 0000000..8b5130c
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java
@@ -0,0 +1,349 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.cli.tools;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Parser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.ActiveRoleSet;
+import org.apache.sentry.core.common.utils.PolicyFileConstants;
+import org.apache.sentry.core.common.utils.PolicyFiles;
+import org.apache.sentry.core.common.utils.Version;
+import org.apache.sentry.policy.common.PrivilegeUtils;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
+import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege;
+import org.apache.sentry.provider.db.generic.service.thrift.TSentryRole;
+import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.apache.shiro.config.Ini;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+
+/**
+ * This class provides basic framework required to migrate permissions between different Sentry
+ * versions. Individual components (e.g. SOLR, KAFKA) needs to override the this class
+ * to provide component specific migration functionality.
+ */
+public abstract class PermissionsMigrationToolCommon {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PermissionsMigrationToolCommon.class);
+ public static final String SOLR_SERVICE_NAME = "sentry.service.client.solr.service.name";
+
+ private Version sourceVersion;
+ private Optional<String> confPath = Optional.empty();
+ private Optional<String> policyFile = Optional.empty();
+ private Optional<String> outputFile = Optional.empty();
+ private boolean dryRun = false;
+
+ /**
+ * @return version of Sentry for which the privileges need to be migrated.
+ */
+ public final Version getSourceVersion() {
+ return sourceVersion;
+ }
+
+ /**
+ * This method returns the name of the component for the migration purpose.
+ * @param conf The Sentry configuration
+ * @return the name of the component
+ */
+ protected abstract String getComponent(Configuration conf);
+
+
+ /**
+ * This method returns the name of the service name for the migration purpose.
+ *
+ * @param conf The Sentry configuration
+ * @return the name of the service
+ */
+ protected abstract String getServiceName(Configuration conf);
+
+ /**
+ * Migrate the privileges specified via <code>privileges</code>.
+ *
+ * @param privileges A collection of privileges to be migrated.
+ * @return A collection of migrated privileges
+ * An empty collection if migration is not necessary for the specified privileges.
+ */
+ protected abstract Collection<String> transformPrivileges (Collection<String> privileges);
+
+ /**
+ * parse arguments
+ * <pre>
+ * -s,--source Sentry source version
+ * -c,--sentry_conf <filepath> sentry config file path
+ * -p --policy_file <filepath> sentry (source) policy file path
+ * -o --output <filepath> sentry (target) policy file path
+ * -d --dry_run provides the output the migration for inspection without
+ * making any configuration changes.
+ * -h,--help print usage
+ * </pre>
+ * @param args
+ */
+ protected boolean parseArgs(String [] args) {
+ Options options = new Options();
+
+ Option sourceVersionOpt = new Option("s", "source", true, "Source Sentry version");
+ sourceVersionOpt.setRequired(true);
+ options.addOption(sourceVersionOpt);
+
+ Option sentryConfPathOpt = new Option("c", "sentry_conf", true,
+ "sentry-site.xml file path (only required in case of Sentry service)");
+ sentryConfPathOpt.setRequired(false);
+ options.addOption(sentryConfPathOpt);
+
+ Option sentryPolicyFileOpt = new Option("p", "policy_file", true,
+ "sentry (source) policy file path (only in case of file based Sentry configuration)");
+ sentryPolicyFileOpt.setRequired(false);
+ options.addOption(sentryPolicyFileOpt);
+
+ Option sentryOutputFileOpt = new Option("o", "output", true,
+ "sentry (target) policy file path (only in case of file based Sentry configuration)");
+ sentryOutputFileOpt.setRequired(false);
+ options.addOption(sentryOutputFileOpt);
+
+ Option dryRunOpt = new Option("d", "dry_run", false,
+ "provides the output the migration for inspection without making actual configuration changes");
+ dryRunOpt.setRequired(false);
+ options.addOption(dryRunOpt);
+
+ // help option
+ Option helpOpt = new Option("h", "help", false, "Shell usage");
+ helpOpt.setRequired(false);
+ options.addOption(helpOpt);
+
+ // this Option is parsed first for help option
+ Options helpOptions = new Options();
+ helpOptions.addOption(helpOpt);
+
+ try {
+ Parser parser = new GnuParser();
+
+ // parse help option first
+ CommandLine cmd = parser.parse(helpOptions, args, true);
+ for (Option opt : cmd.getOptions()) {
+ if (opt.getOpt().equals("h")) {
+ // get the help option, print the usage and exit
+ usage(options);
+ return false;
+ }
+ }
+
+ // without help option
+ cmd = parser.parse(options, args);
+
+ String sourceVersionStr = null;
+
+ for (Option opt : cmd.getOptions()) {
+ if (opt.getOpt().equals("s")) {
+ sourceVersionStr = opt.getValue();
+ } else if (opt.getOpt().equals("c")) {
+ confPath = Optional.of(opt.getValue());
+ } else if (opt.getOpt().equals("p")) {
+ policyFile = Optional.of(opt.getValue());
+ } else if (opt.getOpt().equals("o")) {
+ outputFile = Optional.of(opt.getValue());
+ } else if (opt.getOpt().equals("d")) {
+ dryRun = true;
+ }
+ }
+
+ sourceVersion = Version.parse(sourceVersionStr);
+
+ if (!(confPath.isPresent() || policyFile.isPresent())) {
+ System.out.println("Please select either file-based Sentry configuration (-p and -o flags)"
+ + " or Sentry service (-c flag) for migration.");
+ usage(options);
+ return false;
+ }
+
+ if (confPath.isPresent() && (policyFile.isPresent() || outputFile.isPresent())) {
+ System.out.println("In order to migrate service based Sentry configuration,"
+ + " do not specify either -p or -o parameters");
+ usage(options);
+ return false;
+ }
+
+ if (!confPath.isPresent() && (policyFile.isPresent() ^ outputFile.isPresent())) {
+ System.out.println("In order to migrate file based Sentry configuration,"
+ + " please make sure to specify both -p and -o parameters.");
+ usage(options);
+ return false;
+ }
+
+ } catch (ParseException | java.text.ParseException pe) {
+ System.out.println(pe.getMessage());
+ usage(options);
+ return false;
+ }
+ return true;
+ }
+
+ // print usage
+ private void usage(Options sentryOptions) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("sentryMigrationTool", sentryOptions);
+ }
+
+ public void run() throws Exception {
+ if (policyFile.isPresent()) {
+ migratePolicyFile();
+ } else {
+ migrateSentryServiceConfig();
+ }
+ }
+
+ private void migrateSentryServiceConfig() throws Exception {
+ Configuration conf = getSentryConf();
+ String component = getComponent(conf);
+ String serviceName = getServiceName(conf);
+ GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, serviceName, false);
+
+ // instantiate a client for sentry service. This sets the ugi, so must
+ // be done before getting the ugi below.
+ try(SentryGenericServiceClient client =
+ SentryGenericServiceClientFactory.create(getSentryConf())) {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ String requestorName = ugi.getShortUserName();
+
+ for (TSentryRole r : client.listAllRoles(requestorName, component)) {
+ for (TSentryPrivilege p : client.listAllPrivilegesByRoleName(requestorName,
+ r.getRoleName(), component, serviceName)) {
+
+ String privilegeStr = converter.toString(p);
+ Collection<String> privileges = Collections.singleton(privilegeStr);
+ Collection<String> migrated = transformPrivileges(privileges);
+ if (!migrated.isEmpty()) {
+ LOGGER.info("{} For role {} migrating privileges from {} to {}", getDryRunMessage(), r.getRoleName(),
+ privileges, migrated);
+
+ /*
+ * Note that it is not possible to provide transactional (all-or-nothing) behavior for these configuration
+ * changes since the Sentry client/server protocol does not support. e.g. under certain failure conditions
+ * like crash of Sentry server or network disconnect between client/server, it is possible that the migration
+ * can not complete but can also not be rolled back. Hence this migration tool relies on the fact that privilege
+ * grant/revoke operations are idempotent and hence re-execution of the migration tool will fix any inconsistency
+ * due to such failures.
+ **/
+ boolean originalPermPresent = false;
+ for (String perm : migrated) {
+ if (perm.equalsIgnoreCase(privilegeStr)) {
+ originalPermPresent = true;
+ continue;
+ }
+ TSentryPrivilege x = converter.fromString(perm);
+ LOGGER.info("{} GRANT permission {}", getDryRunMessage(), perm);
+ if (!dryRun) {
+ client.grantPrivilege(requestorName, r.getRoleName(), component, x);
+ }
+ }
+
+ // Revoke old permission (only if not part of migrated permissions)
+ if (!originalPermPresent) {
+ LOGGER.info("{} REVOKE permission {}", getDryRunMessage(), privilegeStr);
+ if (!dryRun) {
+ client.revokePrivilege(requestorName, r.getRoleName(), component, p);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void migratePolicyFile () throws Exception {
+ Configuration conf = getSentryConf();
+ Path sourceFile = new Path (policyFile.get());
+ SimpleFileProviderBackend policyFileBackend = new SimpleFileProviderBackend(conf, sourceFile);
+ ProviderBackendContext ctx = new ProviderBackendContext();
+ policyFileBackend.initialize(ctx);
+
+ Set<String> roles = Sets.newHashSet();
+ Table<String, String, Set<String>> groupRolePrivilegeTable =
+ policyFileBackend.getGroupRolePrivilegeTable();
+
+ Ini output = PolicyFiles.loadFromPath(sourceFile.getFileSystem(conf), sourceFile);
+ Ini.Section rolesSection = output.get(PolicyFileConstants.ROLES);
+
+ for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
+ for (String roleName : policyFileBackend.getRoles(Collections.singleton(groupName), ActiveRoleSet.ALL)) {
+ if (!roles.contains(roleName)) {
+ // Do the actual migration
+ Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
+ Collection<String> migrated = transformPrivileges(privileges);
+
+ if (!migrated.isEmpty()) {
+ LOGGER.info("{} For role {} migrating privileges from {} to {}", getDryRunMessage(),
+ roleName, privileges, migrated);
+ if (!dryRun) {
+ rolesSection.put(roleName, PrivilegeUtils.fromPrivilegeStrings(migrated));
+ }
+ }
+
+ roles.add(roleName);
+ }
+ }
+ }
+
+ if (!dryRun) {
+ Path targetFile = new Path (outputFile.get());
+ PolicyFiles.writeToPath(output, targetFile.getFileSystem(conf), targetFile);
+ LOGGER.info("Successfully saved migrated Sentry policy file at {}", outputFile.get());
+ }
+ }
+
+ private String getDryRunMessage() {
+ return dryRun ? "[Dry Run]" : "";
+ }
+
+ private Configuration getSentryConf() {
+ Configuration conf = new Configuration();
+ if (confPath.isPresent()) {
+ conf.addResource(new Path(confPath.get()), true);
+ }
+ return conf;
+ }
+
+ @VisibleForTesting
+ public boolean executeConfigTool(String [] args) throws Exception {
+ boolean result = true;
+ if (parseArgs(args)) {
+ run();
+ } else {
+ result = false;
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java
new file mode 100644
index 0000000..c24ae93
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.cli.tools;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sentry.core.common.utils.SentryConstants;
+import org.apache.sentry.core.model.solr.validator.SolrPrivilegeValidator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides SOLR specific functionality required for migrating Sentry privileges.
+ */
+public class PermissionsMigrationToolSolr extends PermissionsMigrationToolCommon {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PermissionsMigrationToolSolr.class);
+
+
+ @Override
+ protected String getComponent(Configuration conf) {
+ return "SOLR";
+ }
+
+ @Override
+ protected String getServiceName(Configuration conf) {
+ return conf.get(SOLR_SERVICE_NAME, "service1");
+ }
+
+ @Override
+ protected Collection<String> transformPrivileges(Collection<String> privileges) {
+ List<String> result = new ArrayList<>();
+ boolean migrated = false;
+
+ if (getSourceVersion().major == 1) { // Migrate only Sentry 1.x permissions
+ for (String p : privileges) {
+ SolrPrivilegeValidator v = new SolrPrivilegeValidator();
+ v.validate(p, false);
+
+ if ("collection".equalsIgnoreCase(v.getEntityType()) && "admin".equalsIgnoreCase(v.getEntityName())) {
+ result.add(getPermissionStr("admin", "collections", v.getActionName()));
+ result.add(getPermissionStr("admin", "cores", v.getActionName()));
+ migrated = true;
+ } else if ("collection".equalsIgnoreCase(v.getEntityType()) && "*".equals(v.getEntityName())) {
+ result.add(getPermissionStr("admin", "collections", v.getActionName()));
+ result.add(getPermissionStr("admin", "cores", v.getActionName()));
+ result.add(p);
+ migrated = true;
+ } else {
+ result.add(p);
+ }
+ }
+ }
+
+ return migrated ? result : Collections.emptyList();
+ }
+
+ private String getPermissionStr (String entityType, String entityName, String action) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(entityType);
+ builder.append(SentryConstants.KV_SEPARATOR);
+ builder.append(entityName);
+ if (action != null) {
+ builder.append(SentryConstants.AUTHORIZABLE_SEPARATOR);
+ builder.append(SentryConstants.PRIVILEGE_NAME);
+ builder.append(SentryConstants.KV_SEPARATOR);
+ builder.append(action);
+ }
+ return builder.toString();
+ }
+
+ public static void main(String[] args) throws Exception {
+ PermissionsMigrationToolSolr solrTool = new PermissionsMigrationToolSolr();
+ try {
+ solrTool.executeConfigTool(args);
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage(), e);
+ Throwable current = e;
+ // find the first printable message;
+ while (current != null && current.getMessage() == null) {
+ current = current.getCause();
+ }
+ String error = "";
+ if (current != null && current.getMessage() != null) {
+ error = "Message: " + current.getMessage();
+ }
+ System.out.println("The operation failed. " + error);
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java
new file mode 100644
index 0000000..2d4f973
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.cli.tools;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Parser;
+
+abstract public class SentryConfigToolCommon {
+ private String policyFile;
+ private boolean validate;
+ private boolean importPolicy;
+ private boolean checkCompat;
+ private String confPath;
+
+ /**
+ * parse arguments
+ * <pre>
+ * -conf,--sentry_conf <filepath> sentry config file path
+ * -p,--policy_ini <arg> policy file path
+ * -v,--validate validate policy file
+ * -c,--checkcompat check compatibility with service
+ * -i,--import import policy file
+ * -h,--help print usage
+ * </pre>
+ * @param args
+ */
+ protected boolean parseArgs(String [] args) {
+ Options options = new Options();
+
+ Option globalPolicyPath = new Option("p", "policy_ini", true,
+ "Policy file path");
+ globalPolicyPath.setRequired(true);
+ options.addOption(globalPolicyPath);
+
+ Option validateOpt = new Option("v", "validate", false,
+ "Validate policy file");
+ validateOpt.setRequired(false);
+ options.addOption(validateOpt);
+
+ Option checkCompatOpt = new Option("c","checkcompat",false,
+ "Check compatibility with Sentry Service");
+ checkCompatOpt.setRequired(false);
+ options.addOption(checkCompatOpt);
+
+ Option importOpt = new Option("i", "import", false,
+ "Import policy file");
+ importOpt.setRequired(false);
+ options.addOption(importOpt);
+
+ // file path of sentry-site
+ Option sentrySitePathOpt = new Option("conf", "sentry_conf", true, "sentry-site file path");
+ sentrySitePathOpt.setRequired(true);
+ options.addOption(sentrySitePathOpt);
+
+ // help option
+ Option helpOpt = new Option("h", "help", false, "Shell usage");
+ helpOpt.setRequired(false);
+ options.addOption(helpOpt);
+
+ // this Options is parsed first for help option
+ Options helpOptions = new Options();
+ helpOptions.addOption(helpOpt);
+
+ try {
+ Parser parser = new GnuParser();
+
+ // parse help option first
+ CommandLine cmd = parser.parse(helpOptions, args, true);
+ for (Option opt : cmd.getOptions()) {
+ if (opt.getOpt().equals("h")) {
+ // get the help option, print the usage and exit
+ usage(options);
+ return false;
+ }
+ }
+
+ // without help option
+ cmd = parser.parse(options, args);
+
+ for (Option opt : cmd.getOptions()) {
+ if (opt.getOpt().equals("p")) {
+ policyFile = opt.getValue();
+ } else if (opt.getOpt().equals("v")) {
+ validate = true;
+ } else if (opt.getOpt().equals("i")) {
+ importPolicy = true;
+ } else if (opt.getOpt().equals("c")) {
+ checkCompat = true;
+ } else if (opt.getOpt().equals("conf")) {
+ confPath = opt.getValue();
+ }
+ }
+
+ if (!validate && !importPolicy) {
+ throw new IllegalArgumentException("No action specified; at least one of action or import must be specified");
+ }
+ } catch (ParseException pe) {
+ System.out.println(pe.getMessage());
+ usage(options);
+ return false;
+ }
+ return true;
+ }
+
+ // print usage
+ private void usage(Options sentryOptions) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("sentryConfigTool", sentryOptions);
+ }
+
+ public abstract void run() throws Exception;
+
+ @VisibleForTesting
+ public boolean executeConfigTool(String [] args) throws Exception {
+ boolean result = true;
+ if (parseArgs(args)) {
+ run();
+ } else {
+ result = false;
+ }
+ return result;
+ }
+
+ public String getPolicyFile() { return policyFile; }
+ public boolean getValidate() { return validate; }
+ public boolean getImportPolicy() { return importPolicy; }
+ public boolean getCheckCompat() { return checkCompat; }
+ public String getConfPath() { return confPath; }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java
new file mode 100644
index 0000000..712d9ed
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java
@@ -0,0 +1,341 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.cli.tools;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.Action;
+import org.apache.sentry.core.common.exception.SentryConfigurationException;
+import org.apache.sentry.core.common.utils.KeyValue;
+import org.apache.sentry.core.model.indexer.IndexerPrivilegeModel;
+import org.apache.sentry.provider.common.ProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
+import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.sentry.core.common.utils.SentryConstants.AUTHORIZABLE_SPLITTER;
+import static org.apache.sentry.provider.common.AuthorizationComponent.HBASE_INDEXER;
+import static org.apache.sentry.service.thrift.ServiceConstants.ClientConfig.SERVICE_NAME;
+
+/**
+ * SentryConfigToolIndexer is an administrative tool used to parse a HBase Indexer policy file
+ * and add the role, group mappings, and privileges therein to the Sentry service.
+ */
+public class SentryConfigToolIndexer {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SentryConfigToolIndexer.class);
+
+ private String policyFile;
+ private boolean validate;
+ private boolean importPolicy;
+ private boolean checkCompat;
+ private String confPath;
+
+ private String serviceName;
+
+
+
+ public String getPolicyFile() { return policyFile; }
+
+ public boolean getValidate() { return validate; }
+ public boolean getImportPolicy() { return importPolicy; }
+ public boolean getCheckCompat() { return checkCompat; }
+ public String getConfPath() { return confPath; }
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ /**
+ * Adds command line options for the tool to the passed Options object. Used to extend existing options.
+ * @param options
+ */
+ public void setupOptions(Options options) {
+ Option globalPolicyPath = new Option("f", "policy_ini", true,
+ "Policy file path");
+ globalPolicyPath.setRequired(false);
+ options.addOption(globalPolicyPath);
+
+ Option validateOpt = new Option("v", "validate", false,
+ "Validate policy file");
+ validateOpt.setRequired(false);
+ options.addOption(validateOpt);
+
+ Option checkCompatOpt = new Option("c","checkcompat",false,
+ "Check compatibility with Sentry Service");
+ checkCompatOpt.setRequired(false);
+ options.addOption(checkCompatOpt);
+
+ Option importOpt = new Option("i", "import", false,
+ "Import policy file");
+ importOpt.setRequired(false);
+ options.addOption(importOpt);
+
+ }
+
+ /**
+ * Parses and processes the arguments from the given command line object.
+ * @param cmd
+ */
+ public void parseOptions(CommandLine cmd) {
+ boolean isToolActive = false;
+ for (Option opt : cmd.getOptions()) {
+ if (opt.getOpt().equals("mgr")) {
+ isToolActive = true;
+ }
+ }
+ if (!isToolActive) {
+ return;
+ }
+ for (Option opt : cmd.getOptions()) {
+ if (opt.getOpt().equals("f")) {
+ policyFile = opt.getValue();
+ } else if (opt.getOpt().equals("v")) {
+ validate = true;
+ } else if (opt.getOpt().equals("i")) {
+ importPolicy = true;
+ } else if (opt.getOpt().equals("c")) {
+ checkCompat = true;
+ } else if (opt.getOpt().equals("conf")) {
+ confPath = opt.getValue();
+ } else if (opt.getOpt().equals("s")) {
+ serviceName = opt.getValue();
+ }
+ }
+ if (policyFile == null) {
+ throw new IllegalArgumentException("Missing required option: f");
+ }
+ if (!validate && !importPolicy) {
+ throw new IllegalArgumentException("No action specified; at least one of action or import must be specified");
+ }
+ }
+
+
+ /**
+ * Processes the necessary command based on the arguments parsed earlier.
+ * @throws Exception
+ */
+ public void run() throws Exception {
+ String component = HBASE_INDEXER;
+ Configuration conf = getSentryConf();
+
+ String service = conf.get(SERVICE_NAME, getServiceName());
+
+ if (service == null) {
+ throw new IllegalArgumentException("Service was not defined. Please, use -s command option, or sentry.provider.backend.generic.service-name configuration entry.");
+ }
+
+ LOGGER.info(String.format("Context: component=%s, service=%s", component, service));
+ // instantiate a solr client for sentry service. This sets the ugi, so must
+ // be done before getting the ugi below.
+ try(SentryGenericServiceClient client =
+ SentryGenericServiceClientFactory.create(conf)) {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ String requestorName = ugi.getShortUserName();
+
+ convertINIToSentryServiceCmds(component, service, requestorName, conf, client,
+ getPolicyFile(), getValidate(), getImportPolicy(), getCheckCompat());
+ }
+ }
+
+ private Configuration getSentryConf() {
+ Configuration conf = new Configuration();
+ conf.addResource(new Path(getConfPath()), true);
+ return conf;
+ }
+
+ private void convertINIToSentryServiceCmds(String component,
+ String service, String requestorName,
+ Configuration conf, SentryGenericServiceClient client,
+ String policyFile, boolean validate, boolean importPolicy,
+ boolean checkCompat) throws Exception {
+
+ //instantiate a file providerBackend for parsing
+ LOGGER.info("Reading policy file at: " + policyFile);
+ SimpleFileProviderBackend policyFileBackend =
+ new SimpleFileProviderBackend(conf, policyFile);
+ ProviderBackendContext context = new ProviderBackendContext();
+ context.setValidators(IndexerPrivilegeModel.getInstance().getPrivilegeValidators());
+ policyFileBackend.initialize(context);
+ if (validate) {
+ validatePolicy(policyFileBackend);
+ }
+
+ if (checkCompat) {
+ checkCompat(policyFileBackend);
+ }
+
+ //import the relations about group,role and privilege into the DB store
+ Set<String> roles = Sets.newHashSet();
+ Table<String, String, Set<String>> groupRolePrivilegeTable =
+ policyFileBackend.getGroupRolePrivilegeTable();
+ GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, service, false);
+
+ for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
+ for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+ if (!roles.contains(roleName)) {
+ LOGGER.info(dryRunMessage(importPolicy) + "Creating role: " + roleName.toLowerCase(Locale.US));
+ if (importPolicy) {
+ client.createRoleIfNotExist(requestorName, roleName, component);
+ }
+ roles.add(roleName);
+ }
+
+ Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
+ if (privileges == null) {
+ continue;
+ }
+ LOGGER.info(dryRunMessage(importPolicy) + "Adding role: " + roleName.toLowerCase(Locale.US) + " to group: " + groupName);
+ if (importPolicy) {
+ client.grantRoleToGroups(requestorName, roleName, component, Sets.newHashSet(groupName));
+ }
+
+ for (String permission : privileges) {
+ String action = null;
+
+ for (String authorizable : AUTHORIZABLE_SPLITTER.
+ trimResults().split(permission)) {
+ KeyValue kv = new KeyValue(authorizable);
+ String key = kv.getKey();
+ String value = kv.getValue();
+ if ("action".equalsIgnoreCase(key)) {
+ action = value;
+ }
+ }
+
+ // Service doesn't support not specifying action
+ if (action == null) {
+ permission += "->action=" + Action.ALL;
+ }
+ LOGGER.info(dryRunMessage(importPolicy) + "Adding permission: " + permission + " to role: " + roleName.toLowerCase(Locale.US));
+ if (importPolicy) {
+ client.grantPrivilege(requestorName, roleName, component, converter.fromString(permission));
+ }
+ }
+ }
+ }
+ }
+
+ private void validatePolicy(ProviderBackend backend) throws Exception {
+ try {
+ backend.validatePolicy(true);
+ } catch (SentryConfigurationException e) {
+ printConfigErrorsWarnings(e);
+ throw e;
+ }
+ }
+
+ private void printConfigErrorsWarnings(SentryConfigurationException configException) {
+ System.out.println(" *** Found configuration problems *** ");
+ for (String errMsg : configException.getConfigErrors()) {
+ System.out.println("ERROR: " + errMsg);
+ }
+ for (String warnMsg : configException.getConfigWarnings()) {
+ System.out.println("Warning: " + warnMsg);
+ }
+ }
+
+ private void checkCompat(SimpleFileProviderBackend backend) throws Exception {
+ Map<String, Set<String>> rolesCaseMapping = new HashMap<String, Set<String>>();
+ Table<String, String, Set<String>> groupRolePrivilegeTable =
+ backend.getGroupRolePrivilegeTable();
+
+ for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+ String roleNameLower = roleName.toLowerCase(Locale.US);
+ if (!roleName.equals(roleNameLower)) {
+ if (!rolesCaseMapping.containsKey(roleNameLower)) {
+ rolesCaseMapping.put(roleNameLower, Sets.newHashSet(roleName));
+ } else {
+ rolesCaseMapping.get(roleNameLower).add(roleName);
+ }
+ }
+ }
+
+ List<String> errors = new LinkedList<String>();
+ StringBuilder warningString = new StringBuilder();
+ if (!rolesCaseMapping.isEmpty()) {
+ warningString.append("The following roles names will be lower cased when added to the Sentry Service.\n");
+ warningString.append("This will cause document-level security to fail to match the role tokens.\n");
+ warningString.append("Role names: ");
+ }
+ boolean firstWarning = true;
+
+ for (Map.Entry<String, Set<String>> entry : rolesCaseMapping.entrySet()) {
+ Set<String> caseMapping = entry.getValue();
+ if (caseMapping.size() > 1) {
+ StringBuilder errorString = new StringBuilder();
+ errorString.append("The following (cased) roles map to the same role in the sentry service: ");
+ boolean first = true;
+ for (String casedRole : caseMapping) {
+ errorString.append(first ? "" : ", ");
+ errorString.append(casedRole);
+ first = false;
+ }
+ errorString.append(". Role in service: ").append(entry.getKey());
+ errors.add(errorString.toString());
+ }
+
+ for (String casedRole : caseMapping) {
+ warningString.append(firstWarning? "" : ", ");
+ warningString.append(casedRole);
+ firstWarning = false;
+ }
+ }
+
+ for (String error : errors) {
+ System.out.println("ERROR: " + error);
+ }
+ System.out.println("\n");
+
+ System.out.println("Warning: " + warningString.toString());
+ if (errors.size() > 0) {
+ SentryConfigurationException ex =
+ new SentryConfigurationException("Compatibility check failure");
+ ex.setConfigErrors(errors);
+ ex.setConfigWarnings(Lists.<String>asList(warningString.toString(), new String[0]));
+ throw ex;
+ }
+ }
+
+ private String dryRunMessage(boolean importPolicy) {
+ if (importPolicy) {
+ return "";
+ } else {
+ return "[Dry Run] ";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java
new file mode 100644
index 0000000..dfa981d
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.cli.tools;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.sentry.core.common.Action;
+import org.apache.sentry.core.common.exception.SentryConfigurationException;
+import org.apache.sentry.core.common.utils.KeyValue;
+import org.apache.sentry.core.common.utils.SentryConstants;
+import org.apache.sentry.core.model.solr.SolrPrivilegeModel;
+import org.apache.sentry.provider.common.ProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
+import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
+import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * SentryConfigToolSolr is an administrative tool used to parse a Solr policy file
+ * and add the role, group mappings, and privileges therein to the Sentry service.
+ */
+public class SentryConfigToolSolr extends SentryConfigToolCommon {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SentryConfigToolSolr.class);
+ public static final String SOLR_SERVICE_NAME = "sentry.service.client.solr.service.name";
+
+ @Override
+ public void run() throws Exception {
+ String component = "SOLR";
+ Configuration conf = getSentryConf();
+
+ String service = conf.get(SOLR_SERVICE_NAME, "service1");
+ // instantiate a solr client for sentry service. This sets the ugi, so must
+ // be done before getting the ugi below.
+ try(SentryGenericServiceClient client =
+ SentryGenericServiceClientFactory.create(conf)) {
+ UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+ String requestorName = ugi.getShortUserName();
+
+ convertINIToSentryServiceCmds(component, service, requestorName, conf, client,
+ getPolicyFile(), getValidate(), getImportPolicy(), getCheckCompat());
+ }
+ }
+
+ private Configuration getSentryConf() {
+ Configuration conf = new Configuration();
+ conf.addResource(new Path(getConfPath()), true);
+ return conf;
+ }
+
+ /**
+ * Convert policy file to solrctl commands -- based on SENTRY-480
+ */
+ private void convertINIToSentryServiceCmds(String component,
+ String service, String requestorName,
+ Configuration conf, SentryGenericServiceClient client,
+ String policyFile, boolean validate, boolean importPolicy,
+ boolean checkCompat) throws Exception {
+
+ //instantiate a file providerBackend for parsing
+ LOGGER.info("Reading policy file at: " + policyFile);
+ SimpleFileProviderBackend policyFileBackend =
+ new SimpleFileProviderBackend(conf, policyFile);
+ ProviderBackendContext context = new ProviderBackendContext();
+ context.setValidators(SolrPrivilegeModel.getInstance().getPrivilegeValidators());
+ policyFileBackend.initialize(context);
+ if (validate) {
+ validatePolicy(policyFileBackend);
+ }
+
+ if (checkCompat) {
+ checkCompat(policyFileBackend);
+ }
+
+ //import the relations about group,role and privilege into the DB store
+ Set<String> roles = Sets.newHashSet();
+ Table<String, String, Set<String>> groupRolePrivilegeTable =
+ policyFileBackend.getGroupRolePrivilegeTable();
+ GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, service, false);
+
+ for (String groupName : groupRolePrivilegeTable.rowKeySet()) {
+ for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+ if (!roles.contains(roleName)) {
+ LOGGER.info(dryRunMessage(importPolicy) + "Creating role: " + roleName.toLowerCase(Locale.US));
+ if (importPolicy) {
+ client.createRoleIfNotExist(requestorName, roleName, component);
+ }
+ roles.add(roleName);
+ }
+
+ Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName);
+ if (privileges == null) {
+ continue;
+ }
+ LOGGER.info(dryRunMessage(importPolicy) + "Adding role: " + roleName.toLowerCase(Locale.US) + " to group: " + groupName);
+ if (importPolicy) {
+ client.grantRoleToGroups(requestorName, roleName, component, Sets.newHashSet(groupName));
+ }
+
+ for (String permission : privileges) {
+ String action = null;
+
+ for (String authorizable : SentryConstants.AUTHORIZABLE_SPLITTER.
+ trimResults().split(permission)) {
+ KeyValue kv = new KeyValue(authorizable);
+ String key = kv.getKey();
+ String value = kv.getValue();
+ if ("action".equalsIgnoreCase(key)) {
+ action = value;
+ }
+ }
+
+ // Service doesn't support not specifying action
+ if (action == null) {
+ permission += "->action=" + Action.ALL;
+ }
+ LOGGER.info(dryRunMessage(importPolicy) + "Adding permission: " + permission + " to role: " + roleName.toLowerCase(Locale.US));
+ if (importPolicy) {
+ client.grantPrivilege(requestorName, roleName, component, converter.fromString(permission));
+ }
+ }
+ }
+ }
+ }
+
+ private void validatePolicy(ProviderBackend backend) throws Exception {
+ try {
+ backend.validatePolicy(true);
+ } catch (SentryConfigurationException e) {
+ printConfigErrorsWarnings(e);
+ throw e;
+ }
+ }
+
+ private void printConfigErrorsWarnings(SentryConfigurationException configException) {
+ System.out.println(" *** Found configuration problems *** ");
+ for (String errMsg : configException.getConfigErrors()) {
+ System.out.println("ERROR: " + errMsg);
+ }
+ for (String warnMsg : configException.getConfigWarnings()) {
+ System.out.println("Warning: " + warnMsg);
+ }
+ }
+
+ private void checkCompat(SimpleFileProviderBackend backend) throws Exception {
+ Map<String, Set<String>> rolesCaseMapping = new HashMap<String, Set<String>>();
+ Table<String, String, Set<String>> groupRolePrivilegeTable =
+ backend.getGroupRolePrivilegeTable();
+
+ for (String roleName : groupRolePrivilegeTable.columnKeySet()) {
+ String roleNameLower = roleName.toLowerCase(Locale.US);
+ if (!roleName.equals(roleNameLower)) {
+ if (!rolesCaseMapping.containsKey(roleNameLower)) {
+ rolesCaseMapping.put(roleNameLower, Sets.newHashSet(roleName));
+ } else {
+ rolesCaseMapping.get(roleNameLower).add(roleName);
+ }
+ }
+ }
+
+ List<String> errors = new LinkedList<String>();
+ StringBuilder warningString = new StringBuilder();
+ if (!rolesCaseMapping.isEmpty()) {
+ warningString.append("The following roles names will be lower cased when added to the Sentry Service.\n");
+ warningString.append("This will cause document-level security to fail to match the role tokens.\n");
+ warningString.append("Role names: ");
+ }
+ boolean firstWarning = true;
+
+ for (Map.Entry<String, Set<String>> entry : rolesCaseMapping.entrySet()) {
+ Set<String> caseMapping = entry.getValue();
+ if (caseMapping.size() > 1) {
+ StringBuilder errorString = new StringBuilder();
+ errorString.append("The following (cased) roles map to the same role in the sentry service: ");
+ boolean first = true;
+ for (String casedRole : caseMapping) {
+ errorString.append(first ? "" : ", ");
+ errorString.append(casedRole);
+ first = false;
+ }
+ errorString.append(". Role in service: ").append(entry.getKey());
+ errors.add(errorString.toString());
+ }
+
+ for (String casedRole : caseMapping) {
+ warningString.append(firstWarning? "" : ", ");
+ warningString.append(casedRole);
+ firstWarning = false;
+ }
+ }
+
+ for (String error : errors) {
+ System.out.println("ERROR: " + error);
+ }
+ System.out.println("\n");
+
+ System.out.println("Warning: " + warningString.toString());
+ if (errors.size() > 0) {
+ SentryConfigurationException ex =
+ new SentryConfigurationException("Compatibility check failure");
+ ex.setConfigErrors(errors);
+ ex.setConfigWarnings(Lists.<String>asList(warningString.toString(), new String[0]));
+ throw ex;
+ }
+ }
+
+ private String dryRunMessage(boolean importPolicy) {
+ if (importPolicy) {
+ return "";
+ } else {
+ return "[Dry Run] ";
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ SentryConfigToolSolr solrTool = new SentryConfigToolSolr();
+ try {
+ solrTool.executeConfigTool(args);
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage(), e);
+ Throwable current = e;
+ // find the first printable message;
+ while (current != null && current.getMessage() == null) {
+ current = current.getCause();
+ }
+ String error = "";
+ if (current != null && current.getMessage() != null) {
+ error = "Message: " + current.getMessage();
+ }
+ System.out.println("The operation failed. " + error);
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java
----------------------------------------------------------------------
diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java
new file mode 100644
index 0000000..3723fd9
--- /dev/null
+++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java
@@ -0,0 +1,315 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.cli.tools;
+
+import java.util.IllegalFormatException;
+
+public final class SentrySchemaHelper {
+ public static final String DB_DERBY = "derby";
+ public static final String DB_MYSQL = "mysql";
+ public static final String DB_POSTGRACE = "postgres";
+ public static final String DB_ORACLE = "oracle";
+ public static final String DB_DB2 = "db2";
+
+ public interface NestedScriptParser {
+
+ public enum CommandType {
+ PARTIAL_STATEMENT,
+ TERMINATED_STATEMENT,
+ COMMENT
+ }
+
+ String DEFAUTL_DELIMITER = ";";
+ /***
+ * Find the type of given command
+ * @param dbCommand
+ * @return
+ */
+ boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
+
+ /** Parse the DB specific nesting format and extract the inner script name if any
+ * @param dbCommand command from parent script
+ * @return
+ * @throws IllegalFormatException
+ */
+ String getScriptName(String dbCommand) throws IllegalArgumentException;
+
+ /***
+ * Find if the given command is a nested script execution
+ * @param dbCommand
+ * @return
+ */
+ boolean isNestedScript(String dbCommand);
+
+ /***
+ * Find if the given command is should be passed to DB
+ * @param dbCommand
+ * @return
+ */
+ boolean isNonExecCommand(String dbCommand);
+
+ /***
+ * Get the SQL statement delimiter
+ * @return
+ */
+ String getDelimiter();
+
+ /***
+ * Clear any client specific tags
+ * @return
+ */
+ String cleanseCommand(String dbCommand);
+
+ /***
+ * Does the DB required table/column names quoted
+ * @return
+ */
+ boolean needsQuotedIdentifier();
+
+ /***
+ * Set DB specific options if any
+ * @param dbOps
+ */
+ void setDbOpts(String dbOps);
+ }
+
+
+ /***
+ * Base implemenation of NestedScriptParser
+ * abstractCommandParser.
+ *
+ */
+ private static abstract class AbstractCommandParser implements NestedScriptParser {
+ private String dbOpts = null;
+
+ @Override
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+ if (dbCommand == null || dbCommand.isEmpty()) {
+ throw new IllegalArgumentException("invalid command line " + dbCommand);
+ }
+ String trimmedDbCommand = dbCommand.trim();
+ return !(trimmedDbCommand.endsWith(getDelimiter()) || isNonExecCommand(trimmedDbCommand));
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ return dbCommand.startsWith("--") || dbCommand.startsWith("#");
+ }
+
+ @Override
+ public String getDelimiter() {
+ return DEFAUTL_DELIMITER;
+ }
+
+ @Override
+ public String cleanseCommand(String dbCommand) {
+ // strip off the delimiter
+ if (dbCommand.endsWith(getDelimiter())) {
+ dbCommand = dbCommand.substring(0,
+ dbCommand.length() - getDelimiter().length());
+ }
+ return dbCommand;
+ }
+
+ @Override
+ public boolean needsQuotedIdentifier() {
+ return false;
+ }
+
+ @Override
+ public void setDbOpts(String dbOpts) {
+ this.dbOpts = dbOpts;
+ }
+
+ protected String getDbOpts() {
+ return dbOpts;
+ }
+ }
+
+
+ // Derby commandline parser
+ public static class DerbyCommandParser extends AbstractCommandParser {
+ private static final String DERBY_NESTING_TOKEN = "RUN";
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a script format " + dbCommand);
+ }
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1].replace(";", "").replaceAll("'", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ // Derby script format is RUN '<file>'
+ return dbCommand.startsWith(DERBY_NESTING_TOKEN);
+ }
+ }
+
+
+ // MySQL parser
+ public static class MySqlCommandParser extends AbstractCommandParser {
+ private static final String MYSQL_NESTING_TOKEN = "SOURCE";
+ private static final String DELIMITER_TOKEN = "DELIMITER";
+ private String delimiter = DEFAUTL_DELIMITER;
+
+ @Override
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+ boolean isPartial = super.isPartialCommand(dbCommand);
+ // if this is a delimiter directive, reset our delimiter
+ if (dbCommand.startsWith(DELIMITER_TOKEN)) {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ delimiter = tokens[1];
+ }
+ return isPartial;
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ // remove ending ';'
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
+ }
+
+ @Override
+ public String getDelimiter() {
+ return delimiter;
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ return super.isNonExecCommand(dbCommand) ||
+ dbCommand.startsWith("/*") && dbCommand.endsWith("*/") ||
+ dbCommand.startsWith(DELIMITER_TOKEN);
+ }
+
+ @Override
+ public String cleanseCommand(String dbCommand) {
+ return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
+ }
+
+ }
+
+ // Postgres specific parser
+ public static class PostgresCommandParser extends AbstractCommandParser {
+ public static final String POSTGRES_STRING_COMMAND_FILTER = "SET standard_conforming_strings";
+ public static final String POSTGRES_STRING_CLIENT_ENCODING = "SET client_encoding";
+ public static final String POSTGRES_SKIP_STANDARD_STRING = "postgres.filter.81";
+ private static final String POSTGRES_NESTING_TOKEN = "\\i";
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ // remove ending ';'
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
+ }
+
+ @Override
+ public boolean needsQuotedIdentifier() {
+ return true;
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ // Skip "standard_conforming_strings" command which is not supported in older postgres
+ if (POSTGRES_SKIP_STANDARD_STRING.equalsIgnoreCase(getDbOpts())
+ && (dbCommand.startsWith(POSTGRES_STRING_COMMAND_FILTER) || dbCommand.startsWith(POSTGRES_STRING_CLIENT_ENCODING))) {
+ return true;
+ }
+ return super.isNonExecCommand(dbCommand);
+ }
+ }
+
+ //Oracle specific parser
+ public static class OracleCommandParser extends AbstractCommandParser {
+ private static final String ORACLE_NESTING_TOKEN = "@";
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a nested script format " + dbCommand);
+ }
+ // remove ending ';' and starting '@'
+ return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
+ }
+ }
+
+ // DB2 commandline parser
+ public static class DB2CommandParser extends AbstractCommandParser {
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ //DB2 does not support nesting script
+ throw new IllegalArgumentException("DB2 does not support nesting script " + dbCommand);
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ //DB2 does not support nesting script
+ return false;
+ }
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName) {
+ if (dbName.equalsIgnoreCase(DB_DERBY)) {
+ return new DerbyCommandParser();
+ } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
+ return new MySqlCommandParser();
+ } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
+ return new PostgresCommandParser();
+ } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
+ return new OracleCommandParser();
+ } else if (dbName.equalsIgnoreCase(DB_DB2)) {
+ return new DB2CommandParser();
+ } else {
+ throw new IllegalArgumentException("Unknown dbType " + dbName);
+ }
+ }
+
+ private SentrySchemaHelper() {
+ // Make constructor private to avoid instantiation
+ }
+}