You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sp...@apache.org on 2018/09/13 16:29:54 UTC
sentry git commit: SENTRY-2398: Support multiple target versions on
single source versions during schema upgrades (Sergio Pena, reviewed by Na Li)
Repository: sentry
Updated Branches:
refs/heads/master 1fbb0aa41 -> 65eceff08
SENTRY-2398: Support multiple target versions on single source versions during schema upgrades (Sergio Pena, reviewed by Na Li)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/65eceff0
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/65eceff0
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/65eceff0
Branch: refs/heads/master
Commit: 65eceff0863bc1a06a5c09c62b46e97675ca7421
Parents: 1fbb0aa
Author: Sergio Pena <se...@cloudera.com>
Authored: Thu Sep 13 11:29:16 2018 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Thu Sep 13 11:29:16 2018 -0500
----------------------------------------------------------------------
.../persistent/SentryStoreSchemaInfo.java | 45 ++---
.../service/persistent/SentryUpgradeOrder.java | 167 +++++++++++++++++
.../persistent/TestSentryUpgradeOrder.java | 187 +++++++++++++++++++
3 files changed, 372 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/65eceff0/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStoreSchemaInfo.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStoreSchemaInfo.java
index 0d4f26c..f710253 100644
--- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStoreSchemaInfo.java
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStoreSchemaInfo.java
@@ -18,14 +18,15 @@
package org.apache.sentry.provider.db.service.persistent;
-import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import org.apache.sentry.core.common.exception.SentryUserException;
public class SentryStoreSchemaInfo {
@@ -34,7 +35,7 @@ public class SentryStoreSchemaInfo {
private static final String INIT_FILE_PREFIX = "sentry-";
private static final String VERSION_UPGRADE_LIST = "upgrade.order";
private final String dbType;
- private final String sentrySchemaVersions[];
+ private final Map<String, List<String>> sentrySchemaVersions;
private final String sentryScriptDir;
private static final String SENTRY_VERSION = "2.1.0";
@@ -43,21 +44,18 @@ public class SentryStoreSchemaInfo {
throws SentryUserException {
this.sentryScriptDir = sentryScriptDir;
this.dbType = dbType;
+
// load upgrade order for the given dbType
- List<String> upgradeOrderList = new ArrayList<String>();
String upgradeListFile = getSentryStoreScriptDir() + File.separator
+ VERSION_UPGRADE_LIST + "." + dbType;
- try (BufferedReader bfReader = new BufferedReader(new FileReader(upgradeListFile))) {
- String currSchemaVersion;
- while ((currSchemaVersion = bfReader.readLine()) != null) {
- upgradeOrderList.add(currSchemaVersion.trim());
- }
+
+ try {
+ sentrySchemaVersions = SentryUpgradeOrder.readUpgradeGraph(new FileReader(upgradeListFile));
} catch (FileNotFoundException e) {
throw new SentryUserException("File " + upgradeListFile + " not found ", e);
} catch (IOException e) {
throw new SentryUserException("Error reading " + upgradeListFile, e);
}
- sentrySchemaVersions = upgradeOrderList.toArray(new String[0]);
}
public String getSentrySchemaVersion() {
@@ -66,31 +64,24 @@ public class SentryStoreSchemaInfo {
public List<String> getUpgradeScripts(String fromSchemaVer)
throws SentryUserException {
- List<String> upgradeScriptList = new ArrayList<String>();
-
// check if we are already at current schema level
if (getSentryVersion().equals(fromSchemaVer)) {
- return upgradeScriptList;
+ return Collections.emptyList();
}
- // Find the list of scripts to execute for this upgrade
- int firstScript = sentrySchemaVersions.length;
- for (int i = 0; i < sentrySchemaVersions.length; i++) {
- String fromVersion = sentrySchemaVersions[i].split("-to-")[0];
- if (fromVersion.equals(fromSchemaVer)) {
- firstScript = i;
- break;
- }
- }
- if (firstScript == sentrySchemaVersions.length) {
+ List<String> upgradePathList =
+ SentryUpgradeOrder.getUpgradePath(sentrySchemaVersions, fromSchemaVer, getSentrySchemaVersion());
+ if (upgradePathList.isEmpty()) {
throw new SentryUserException("Unknown version specified for upgrade "
- + fromSchemaVer + " Sentry schema may be too old or newer");
+ + fromSchemaVer + " Sentry schema may be too old or newer");
}
- for (int i = firstScript; i < sentrySchemaVersions.length; i++) {
- String scriptFile = generateUpgradeFileName(sentrySchemaVersions[i]);
- upgradeScriptList.add(scriptFile);
+ // Create a new list with the script file paths of the upgrade order path obtained before
+ List<String> upgradeScriptList = new LinkedList<>();
+ for (String upgradePath : upgradePathList) {
+ upgradeScriptList.add(generateUpgradeFileName(upgradePath));
}
+
return upgradeScriptList;
}
http://git-wip-us.apache.org/repos/asf/sentry/blob/65eceff0/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryUpgradeOrder.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryUpgradeOrder.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryUpgradeOrder.java
new file mode 100644
index 0000000..a7a3392
--- /dev/null
+++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryUpgradeOrder.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.provider.db.service.persistent;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SentryUpgradeOrder {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SentryUpgradeOrder.class);
+
+ /**
+ * Returns a directed graph in the form of a map structure that contains the allowed upgrade
+ * order schema versions read from a specific Reader.
+ * </p>
+ * The format of the upgrade order schema versions is as follows:
+ * SOURCE-to-TARGET
+ * </p>
+ * where SOURCE is the version from where to upgrade, and TARGET is the target version where
+ * to upgrade, i.e.
+ * 1.8.0-to-1.9.0
+ * </p>
+ * The reader may contain a list of upgrade path versions separated by a new line.
+ * 1.8.0-to-1.9.0
+ * 1.8.0-to-2.0.0
+ * 1.9.0-to-2.1.0
+ * </p>
+ * Lines starting with the # are considered comments and are ignored.
+ *
+ * @param reader A Reader object from where to read the list of upgrade path versions.
+ * @return A map structure that contains the directed graph.
+ * @throws IOException If an error occurs while reading data from the Reader object.
+ */
+ public static Map<String, List<String>> readUpgradeGraph(Reader reader) throws IOException {
+ Map<String, List<String>> upgradeSchemaVersions = new HashMap<>();
+
+ try (BufferedReader bfReader = new BufferedReader(reader)) {
+ String schemaVersion;
+ while ((schemaVersion = bfReader.readLine()) != null) {
+ String[] versions = schemaVersion.toLowerCase().split("-to-");
+
+ // Valid upgrade versions has a source and a target version
+ if (versions.length != 2) {
+ LOGGER.warn("Ignoring unknown Sentry schema upgrade path: " + schemaVersion);
+ continue;
+ }
+
+ String source = versions[0].trim();
+ String target = versions[1].trim();
+
+ // Valid upgrade versions should have different source and target versions
+ if (source.isEmpty() || source.startsWith("#") || source.equals(target)) {
+ continue;
+ }
+
+ // There could be multiple source versions with different targets.
+ List<String> toVersions = upgradeSchemaVersions.getOrDefault(source, new LinkedList<>());
+ toVersions.add(target);
+ upgradeSchemaVersions.put(source, toVersions);
+ }
+ }
+
+ return upgradeSchemaVersions;
+ }
+
+ /**
+ * Returns a list of upgrade path versions sorted from lower version to higher version.
+ *
+ * @param upgrades The map structured that contains a graph of directed path upgrades.
+ * @param from A string of the source version to search.
+ * @param to A string of the target version to search.
+ * @return An ordered list from lower to higher versions of the upgrade path versions.
+ */
+ public static List<String> getUpgradePath(Map<String, List<String>> upgrades, String from, String to) {
+ // LinkedList is used to keep the correct order in Java 8 (ArrayList is known to not preserve
+ // the order at the moment of insertion).
+ List<String> upgradeListPath = new LinkedList<>();
+
+ Stack<String> upgradePaths = new Stack<>();
+ Set<String> visited = new HashSet<>();
+ searchPath(upgrades, from, to, upgradePaths, visited);
+
+ // Reverse the stack to put the correct order in the list.
+ // To keep the behavior correctly, a list is returned instead of the stack to avoid the
+ // incorrect order obtained if a stack is used in the for() loop.
+ while (!upgradePaths.isEmpty()) {
+ upgradeListPath.add(upgradePaths.pop());
+ }
+
+ return upgradeListPath;
+ }
+
+ /**
+ * Searches the complete upgrade path from the list of valid upgrades.
+ *
+ * @param from The string version from where to upgrade.
+ * @param to The string version to upgrade.
+ * @param upgradePath A stack where to put the complete upgrade path. Empty if no upgrade path
+ * is found.
+ * @param visited A set where to track the visited upgrade paths.
+ *
+ * @return True if a complete upgrade path is found; False otherwise.
+ */
+ private static boolean searchPath(Map<String, List<String>> graph, String from, String to, Stack<String> upgradePath, Set<String> visited) {
+ /*
+ * It is not necessary to find the shortest path to the upgrade as the number of upgrade
+ * scripts are small and do not cause too much overhead during an upgrade. So a simple depth-first
+ * search is done to find the requested path.
+ */
+
+ if (from.equals(to)) {
+ return true;
+ }
+
+ if (!graph.containsKey(from)) {
+ return false;
+ }
+
+ boolean found = false;
+ for (String toVersion : graph.get(from)) {
+ String nextPath = generateUpgradeVersionString(from, toVersion);
+ boolean isVisited = visited.contains(nextPath);
+ if (!isVisited) {
+ visited.add(nextPath);
+ found = searchPath(graph, toVersion, to, upgradePath, visited);
+ if (found) {
+ upgradePath.push(nextPath);
+ break;
+ } else {
+ visited.remove(nextPath);
+ }
+ }
+ }
+
+ return found;
+ }
+
+ private static String generateUpgradeVersionString(String from, String to) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(from).append("-to-").append(to);
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/65eceff0/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryUpgradeOrder.java
----------------------------------------------------------------------
diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryUpgradeOrder.java b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryUpgradeOrder.java
new file mode 100644
index 0000000..4d0141e
--- /dev/null
+++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryUpgradeOrder.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.provider.db.service.persistent;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang.StringUtils;
+import org.junit.Test;
+
+public class TestSentryUpgradeOrder {
+ @Test
+ public void testReadUpgradeOrderGraphInNormalOrder() throws IOException {
+ List<String> normalOrder = Arrays.asList(
+ "1.7.0-to-1.8.0",
+ "1.8.0-to-2.0.0",
+ "2.0.0-to-2.1.0"
+ );
+
+ Map<String, List<String>> upgradeGraph =
+ SentryUpgradeOrder.readUpgradeGraph(new StringReader(StringUtils.join(normalOrder, "\n")));
+
+ // The returned graph should be in the following form:
+ // key: 1.7.0 value: 1.8.0
+ // key: 1.8.0 value: 2.0.0
+ // key: 2.0.0 value: 2.1.0
+ assertEquals(3, upgradeGraph.size());
+ assertEquals(Arrays.asList("1.8.0"), upgradeGraph.get("1.7.0"));
+ assertEquals(Arrays.asList("2.0.0"), upgradeGraph.get("1.8.0"));
+ assertEquals(Arrays.asList("2.1.0"), upgradeGraph.get("2.0.0"));
+ }
+
+ @Test
+ public void testReadUpgradeOrderGraphInReverseOrder() throws IOException {
+ List<String> simpleOrder = Arrays.asList(
+ "2.0.0-to-2.1.0",
+ "1.8.0-to-2.0.0",
+ "1.7.0-to-1.8.0"
+ );
+
+ Map<String, List<String>> upgradeGraph =
+ SentryUpgradeOrder.readUpgradeGraph(new StringReader(StringUtils.join(simpleOrder, "\n")));
+
+ // The returned graph should be in the following form:
+ // key: 1.7.0 value: 1.8.0
+ // key: 1.8.0 value: 2.0.0
+ // key: 2.0.0 value: 2.1.0
+ assertEquals(3, upgradeGraph.size());
+ assertEquals(Arrays.asList("1.8.0"), upgradeGraph.get("1.7.0"));
+ assertEquals(Arrays.asList("2.0.0"), upgradeGraph.get("1.8.0"));
+ assertEquals(Arrays.asList("2.1.0"), upgradeGraph.get("2.0.0"));
+ }
+
+ @Test
+ public void testReadUpgradeOrderGraphWithSameSourceVersions() throws IOException {
+ List<String> normalOrder = Arrays.asList(
+ "1.7.0-to-1.8.0",
+ "1.7.0-to-2.0.0",
+ "1.8.0-to-2.0.0",
+ "2.0.0-to-2.1.0",
+ "2.0.0-to-2.2.0"
+ );
+
+ Map<String, List<String>> upgradeGraph =
+ SentryUpgradeOrder.readUpgradeGraph(new StringReader(StringUtils.join(normalOrder, "\n")));
+
+ // The returned graph should be in the following form:
+ // key: 1.7.0 value: 1.8.0, 2.0.0
+ // key: 1.8.0 value: 2.0.0
+ // key: 2.0.0 value: 2.1.0, 2.2.0
+ assertEquals(3, upgradeGraph.size());
+ assertEquals(Arrays.asList("1.8.0", "2.0.0"), upgradeGraph.get("1.7.0"));
+ assertEquals(Arrays.asList("2.0.0"), upgradeGraph.get("1.8.0"));
+ assertEquals(Arrays.asList("2.1.0", "2.2.0"), upgradeGraph.get("2.0.0"));
+ }
+
+ @Test
+ public void testReadUpgradeOrderGraphWithIgnoredVersions() throws IOException {
+ List<String> simpleOrder = Arrays.asList(
+ "# comment", // comments are ignored
+ "# 1.8.0-to-1.9.0",// comments are ignored
+ "2.0.0-to-2.0.0", // same source/target versions are ignored
+ "1.8.0-to-", // no target versions are ignored
+ "-to-1.8.0", // no source versions are ignored
+ "1.8.0-1.7.0", // versions without -to- are ignored
+ "1.8.0-to-1.7.0-to-2.0.0", // more than two versions are ignored
+ "1.7.0-to-1.8.0"
+ );
+
+ Map<String, List<String>> upgradeGraph =
+ SentryUpgradeOrder.readUpgradeGraph(new StringReader(StringUtils.join(simpleOrder, "\n")));
+
+ // The returned graph should be in the following form:
+ // key: 1.7.0 value: 1.8.0
+ assertEquals(1, upgradeGraph.size());
+ assertEquals(Arrays.asList("1.8.0"), upgradeGraph.get("1.7.0"));
+ }
+
+ @Test
+ public void testGetUpgradePathUsingSingleSourceVersions() {
+ Map<String, List<String>> normalOrder = new HashMap<String, List<String>>() {{
+ put("1.7.0", Arrays.asList("1.8.0"));
+ put("1.8.0", Arrays.asList("2.0.0"));
+ put("2.0.0", Arrays.asList("2.1.0"));
+ }};
+
+ List<String> upgradePath;
+
+ // The returned upgrade path should be: 1.8.0 -> 2.0.0 -> 2.1.0
+ upgradePath = SentryUpgradeOrder.getUpgradePath(normalOrder, "1.8.0", "2.1.0");
+ assertEquals(2, upgradePath.size());
+ assertEquals("1.8.0-to-2.0.0", upgradePath.get(0));
+ assertEquals("2.0.0-to-2.1.0", upgradePath.get(1));
+
+ // The returned upgrade path should be: 1.7.0 -> 1.8.0 -> 2.0.0
+ upgradePath = SentryUpgradeOrder.getUpgradePath(normalOrder, "1.7.0", "2.0.0");
+ assertEquals(2, upgradePath.size());
+ assertEquals("1.7.0-to-1.8.0", upgradePath.get(0));
+ assertEquals("1.8.0-to-2.0.0", upgradePath.get(1));
+
+ // This upgrade path does not exist
+ upgradePath = SentryUpgradeOrder.getUpgradePath(normalOrder, "1.8.0", "3.1.0");
+ assertEquals(0, upgradePath.size());
+
+ // This upgrade path does not exist
+ upgradePath = SentryUpgradeOrder.getUpgradePath(normalOrder, "1.5.0", "2.1.0");
+ assertEquals(0, upgradePath.size());
+ }
+
+ @Test
+ public void testGetUpgradePathUsingMultipleSourceTargetVersions() {
+ Map<String, List<String>> multiVersionsOrder = new HashMap<String, List<String>>() {{
+ put("1.7.0", Arrays.asList("1.8.0", "2.0.0"));
+ put("1.8.0", Arrays.asList("2.0.0"));
+ put("1.9.0", Arrays.asList("2.1.0"));
+ put("2.0.0", Arrays.asList("2.1.0"));
+ }};
+
+ List<String> upgradePath;
+
+ // The returned upgrade path should be: 1.8.0 -> 2.0.0 -> 2.1.0
+ upgradePath = SentryUpgradeOrder.getUpgradePath(multiVersionsOrder, "1.8.0", "2.1.0");
+ assertEquals(2, upgradePath.size());
+ assertEquals("1.8.0-to-2.0.0", upgradePath.get(0));
+ assertEquals("2.0.0-to-2.1.0", upgradePath.get(1));
+
+ // The returned upgrade path should be: 1.7.0 -> 1.8.0 -> 2.0.0
+ upgradePath = SentryUpgradeOrder.getUpgradePath(multiVersionsOrder, "1.7.0", "2.0.0");
+ assertEquals(2, upgradePath.size());
+ assertEquals("1.7.0-to-1.8.0", upgradePath.get(0));
+ assertEquals("1.8.0-to-2.0.0", upgradePath.get(1));
+
+ // The returned upgrade path should be: 1.9.0 -> 2.1.0
+ upgradePath = SentryUpgradeOrder.getUpgradePath(multiVersionsOrder, "1.9.0", "2.1.0");
+ assertEquals(1, upgradePath.size());
+
+ // The returned upgrade path should be: 1.7.0 -> 1.8.0 -> 2.0.0
+ upgradePath = SentryUpgradeOrder.getUpgradePath(multiVersionsOrder, "1.7.0", "2.0.0");
+ assertEquals(2, upgradePath.size());
+ assertEquals("1.7.0-to-1.8.0", upgradePath.get(0));
+ assertEquals("1.8.0-to-2.0.0", upgradePath.get(1));
+
+ // This upgrade path does not exist
+ upgradePath = SentryUpgradeOrder.getUpgradePath(multiVersionsOrder, "1.9.0", "2.0.0");
+ assertEquals(0, upgradePath.size());
+ }
+}