You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by el...@apache.org on 2017/04/05 22:38:10 UTC
[1/3] phoenix git commit: PHOENIX-3756 Handle users lacking ADMIN for
the SYSTEM namespace
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-0.98 12fd0fa46 -> 943ddfa6c
refs/heads/4.x-HBase-1.1 f5f00b522 -> 1897fc458
refs/heads/master 2c53fc985 -> 8b3cc71eb
PHOENIX-3756 Handle users lacking ADMIN for the SYSTEM namespace
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8b3cc71e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8b3cc71e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8b3cc71e
Branch: refs/heads/master
Commit: 8b3cc71eb9ae5972516435629591dd2ab94df50d
Parents: 2c53fc9
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 30 15:13:57 2017 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Apr 5 17:48:20 2017 -0400
----------------------------------------------------------------------
.../end2end/SystemTablePermissionsIT.java | 263 +++++++++++++++++++
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +
.../query/ConnectionQueryServicesImpl.java | 56 +++-
.../query/ConnectionQueryServicesImplTest.java | 73 +++++
4 files changed, 383 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b3cc71e/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
new file mode 100644
index 0000000..9f213c8
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.security.access.AccessControlClient;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.phoenix.query.QueryServices;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test that verifies a user can read Phoenix tables with a minimal set of permissions.
+ */
+public class SystemTablePermissionsIT {
+ private static String SUPERUSER;
+
+ private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
+ "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
+ "SYSTEM.MUTEX"));
+ // PHOENIX-XXXX SYSTEM.MUTEX isn't being created in the SYSTEM namespace as it should be.
+ private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
+ Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
+ "SYSTEM.MUTEX"));
+
+ private static final String TABLE_NAME =
+ SystemTablePermissionsIT.class.getSimpleName().toUpperCase();
+ private static final int NUM_RECORDS = 5;
+
+ private HBaseTestingUtility testUtil = null;
+ private Properties clientProperties = null;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ SUPERUSER = System.getProperty("user.name");
+ }
+
+ private static void setCommonConfigProperties(Configuration conf) {
+ conf.set("hbase.coprocessor.master.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.coprocessor.region.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.coprocessor.regionserver.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.security.exec.permission.checks", "true");
+ conf.set("hbase.security.authorization", "true");
+ conf.set("hbase.superuser", SUPERUSER);
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ if (null != testUtil) {
+ testUtil.shutdownMiniCluster();
+ testUtil = null;
+ }
+ }
+
+ @Test
+ public void testSystemTablePermissions() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ clientProperties = new Properties();
+ Configuration conf = testUtil.getConfiguration();
+ setCommonConfigProperties(conf);
+ conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "false");
+ clientProperties.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "false");
+ testUtil.startMiniCluster(1);
+ final UserGroupInformation superUser = UserGroupInformation.createUserForTesting(
+ SUPERUSER, new String[0]);
+ final UserGroupInformation regularUser = UserGroupInformation.createUserForTesting(
+ "user", new String[0]);
+
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ createTable();
+ readTable();
+ return null;
+ }
+ });
+
+ Set<String> tables = getHBaseTables();
+ assertTrue("HBase tables do not include expected Phoenix tables: " + tables,
+ tables.containsAll(PHOENIX_SYSTEM_TABLES));
+
+ // Grant permission to the system tables for the unprivileged user
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ grantPermissions(regularUser.getShortUserName(), PHOENIX_SYSTEM_TABLES,
+ Action.EXEC, Action.READ);
+ grantPermissions(regularUser.getShortUserName(),
+ Collections.singleton(TABLE_NAME), Action.READ);
+ } catch (Throwable e) {
+ if (e instanceof Exception) {
+ throw (Exception) e;
+ } else {
+ throw new Exception(e);
+ }
+ }
+ return null;
+ }
+ });
+
+ // Make sure that the unprivileged user can read the table
+ regularUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // We expect this to not throw an error
+ readTable();
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testNamespaceMappedSystemTables() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ clientProperties = new Properties();
+ Configuration conf = testUtil.getConfiguration();
+ setCommonConfigProperties(conf);
+ testUtil.getConfiguration().set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ clientProperties.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ testUtil.startMiniCluster(1);
+ final UserGroupInformation superUser =
+ UserGroupInformation.createUserForTesting(SUPERUSER, new String[0]);
+ final UserGroupInformation regularUser =
+ UserGroupInformation.createUserForTesting("user", new String[0]);
+
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ createTable();
+ readTable();
+ return null;
+ }
+ });
+
+ Set<String> tables = getHBaseTables();
+ assertTrue("HBase tables do not include expected Phoenix tables: " + tables,
+ tables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
+
+ // Grant permission to the system tables for the unprivileged user
+ // An unprivileged user should only need to be able to Read and eXecute on them.
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ grantPermissions(regularUser.getShortUserName(),
+ PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, Action.EXEC, Action.READ);
+ grantPermissions(regularUser.getShortUserName(),
+ Collections.singleton(TABLE_NAME), Action.READ);
+ } catch (Throwable e) {
+ if (e instanceof Exception) {
+ throw (Exception) e;
+ } else {
+ throw new Exception(e);
+ }
+ }
+ return null;
+ }
+ });
+
+ regularUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // We expect this to not throw an error
+ readTable();
+ return null;
+ }
+ });
+ }
+
+ private String getJdbcUrl() {
+ return "jdbc:phoenix:localhost:" + testUtil.getZkCluster().getClientPort() + ":/hbase";
+ }
+
+ private void createTable() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProperties);
+ Statement stmt = conn.createStatement();) {
+ assertFalse(stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME));
+ assertFalse(stmt.execute("CREATE TABLE " + TABLE_NAME
+ + "(pk INTEGER not null primary key, data VARCHAR)"));
+ try (PreparedStatement pstmt = conn.prepareStatement("UPSERT INTO "
+ + TABLE_NAME + " values(?, ?)")) {
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ pstmt.setInt(1, i);
+ pstmt.setString(2, Integer.toString(i));
+ assertEquals(1, pstmt.executeUpdate());
+ }
+ }
+ conn.commit();
+ }
+ }
+
+ private void readTable() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProperties);
+ Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery("SELECT pk, data FROM " + TABLE_NAME);
+ assertNotNull(rs);
+ int i = 0;
+ while (rs.next()) {
+ assertEquals(i, rs.getInt(1));
+ assertEquals(Integer.toString(i), rs.getString(2));
+ i++;
+ }
+ assertEquals(NUM_RECORDS, i);
+ }
+ }
+
+ private void grantPermissions(String toUser, Set<String> tablesToGrant, Action... actions)
+ throws Throwable {
+ for (String table : tablesToGrant) {
+ AccessControlClient.grant(testUtil.getConnection(), TableName.valueOf(table), toUser,
+ null, null, actions);
+ }
+ }
+
+ private Set<String> getHBaseTables() throws IOException {
+ Set<String> tables = new HashSet<>();
+ for (TableName tn : testUtil.getHBaseAdmin().listTableNames()) {
+ tables.add(tn.getNameAsString());
+ }
+ return tables;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b3cc71e/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index e3a206c..e061406 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -30,6 +30,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.ColumnProjector;
@@ -97,6 +98,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_CATALOG = SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"";
public static final String SYSTEM_CATALOG_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA,
SYSTEM_CATALOG_TABLE);
+ public static final TableName SYSTEM_CATALOG_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_CATALOG_NAME);
public static final byte[] SYSTEM_CATALOG_NAME_BYTES = Bytes.toBytes(SYSTEM_CATALOG_NAME);
public static final String SYSTEM_STATS_TABLE = "STATS";
public static final String SYSTEM_STATS_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_STATS_TABLE);
@@ -305,6 +307,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_MUTEX_TABLE_NAME = "MUTEX";
public static final String SYSTEM_MUTEX_NAME = SchemaUtil.getTableName(QueryConstants.SYSTEM_SCHEMA_NAME, SYSTEM_MUTEX_TABLE_NAME);
+ public static final TableName SYSTEM_MUTEX_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_MUTEX_NAME);
public static final byte[] SYSTEM_MUTEX_NAME_BYTES = Bytes.toBytes(SYSTEM_MUTEX_NAME);
public static final byte[] SYSTEM_MUTEX_FAMILY_NAME_BYTES = TABLE_FAMILY_BYTES;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b3cc71e/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ee9f3d0..b402274 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -121,6 +121,7 @@ import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
import org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator;
+import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -253,6 +254,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -1014,7 +1016,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
QueryServicesOptions.DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE);
}
- private NamespaceDescriptor ensureNamespaceCreated(String schemaName) throws SQLException {
+ void ensureNamespaceCreated(String schemaName) throws SQLException {
SQLException sqlE = null;
try (HBaseAdmin admin = getAdmin()) {
NamespaceDescriptor namespaceDescriptor = null;
@@ -1027,13 +1029,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
namespaceDescriptor = NamespaceDescriptor.create(schemaName).build();
admin.createNamespace(namespaceDescriptor);
}
- return namespaceDescriptor;
+ return;
} catch (IOException e) {
sqlE = ServerUtil.parseServerException(e);
} finally {
if (sqlE != null) { throw sqlE; }
}
- return null; // will never make it here
}
/**
@@ -2445,6 +2446,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
ConnectionQueryServicesImpl.this.upgradeRequired.set(true);
}
+ } catch (PhoenixIOException e) {
+ if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
+ // Pass
+ logger.warn("Could not check for Phoenix SYSTEM tables, assuming they exist and are properly configured");
+ checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, getProps()).getName());
+ success = true;
+ } else {
+ initializationException = e;
+ }
+ return null;
}
if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
createOtherSystemTables(metaConnection);
@@ -2497,8 +2508,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void createSysMutexTable(HBaseAdmin admin) throws IOException, SQLException {
try {
- HTableDescriptor tableDesc = new HTableDescriptor(
- TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME_BYTES));
+ final TableName mutexTableName = TableName.valueOf(
+ PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME_BYTES);
+ List<TableName> systemTables = getSystemTableNames(admin);
+ if (systemTables.contains(mutexTableName)) {
+ logger.debug("System mutex table already appears to exist, not creating it");
+ return;
+ }
+ HTableDescriptor tableDesc = new HTableDescriptor(mutexTableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(
PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES);
columnDesc.setTimeToLive(TTL_FOR_MUTEX); // Let mutex expire after some time
@@ -2516,6 +2533,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ List<TableName> getSystemTableNames(HBaseAdmin admin) throws IOException {
+ return Lists.newArrayList(admin.listTableNames(QueryConstants.SYSTEM_SCHEMA_NAME + "\\..*"));
+ }
+
private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQLException {
try {
metaConnection.createStatement().execute(QueryConstants.CREATE_SEQUENCE_METADATA);
@@ -3081,23 +3102,34 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
}
-
- private void ensureSystemTablesUpgraded(ReadOnlyProps props)
+
+ void ensureSystemTablesUpgraded(ReadOnlyProps props)
throws SQLException, IOException, IllegalArgumentException, InterruptedException {
if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, props)) { return; }
HTableInterface metatable = null;
try (HBaseAdmin admin = getAdmin()) {
- ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ // Namespace-mapping is enabled at this point.
+ try {
+ ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ } catch (PhoenixIOException e) {
+ // We could either:
+ // 1) Not access the NS descriptor. The NS may or may not exist at this point.
+ // 2) We could not create the NS
+ // Regardless of the case 1 or 2, if the NS does not exist, we will error expectedly
+ // below. If the NS does exist and is mapped, the below check will exit gracefully.
+ }
- List<TableName> tableNames = Lists.newArrayList(admin.listTableNames(QueryConstants.SYSTEM_SCHEMA_NAME + "\\..*"));
+ List<TableName> tableNames = getSystemTableNames(admin);
+ // No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
if (tableNames.size() == 0) { return; }
+ // Try to move any remaining tables matching "SYSTEM\..*" into "SYSTEM:"
if (tableNames.size() > 5) {
logger.warn("Expected 5 system tables but found " + tableNames.size() + ":" + tableNames);
}
byte[] mappedSystemTable = SchemaUtil
.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props).getName();
metatable = getTable(mappedSystemTable);
- if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)) {
+ if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)) {
if (!admin.tableExists(mappedSystemTable)) {
UpgradeUtil.mapTableToNamespace(admin, metatable,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props, null, PTableType.SYSTEM,
@@ -3106,9 +3138,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0);
}
- tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
+ tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME);
}
- tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME);
+ tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_MUTEX_HBASE_TABLE_NAME);
for (TableName table : tableNames) {
UpgradeUtil.mapTableToNamespace(admin, metatable, table.getNameAsString(), props, null, PTableType.SYSTEM,
null);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8b3cc71e/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
new file mode 100644
index 0000000..73ddd2d
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Test;
+
+public class ConnectionQueryServicesImplTest {
+ private static final PhoenixIOException PHOENIX_IO_EXCEPTION = new PhoenixIOException(new Exception("Test exception"));
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testExceptionHandlingOnSystemNamespaceCreation() throws Exception {
+ ConnectionQueryServicesImpl cqs = mock(ConnectionQueryServicesImpl.class);
+ // Invoke the real methods for these two calls
+ when(cqs.createSchema(any(List.class), anyString())).thenCallRealMethod();
+ doCallRealMethod().when(cqs).ensureSystemTablesUpgraded(any(ReadOnlyProps.class));
+
+ // Spoof out this call so that ensureSystemTablesUpgrade() will return-fast.
+ when(cqs.getSystemTableNames(any(HBaseAdmin.class))).thenReturn(Collections.<TableName> emptyList());
+
+ // Throw a special exception to check on later
+ doThrow(PHOENIX_IO_EXCEPTION).when(cqs).ensureNamespaceCreated(anyString());
+
+ // Make sure that ensureSystemTablesUpgraded will try to migrate the system tables.
+ Map<String,String> props = new HashMap<>();
+ props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ cqs.ensureSystemTablesUpgraded(new ReadOnlyProps(props));
+
+ // Should be called after upgradeSystemTables()
+ // Proves that execution proceeded
+ verify(cqs).getSystemTableNames(any(HBaseAdmin.class));
+
+ try {
+ // Verifies that the exception is propagated back to the caller
+ cqs.createSchema(Collections.<Mutation> emptyList(), "");
+ } catch (PhoenixIOException e) {
+ assertEquals(PHOENIX_IO_EXCEPTION, e);
+ }
+ }
+}
[3/3] phoenix git commit: PHOENIX-3756 Handle users lacking ADMIN for
the SYSTEM namespace
Posted by el...@apache.org.
PHOENIX-3756 Handle users lacking ADMIN for the SYSTEM namespace
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/943ddfa6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/943ddfa6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/943ddfa6
Branch: refs/heads/4.x-HBase-0.98
Commit: 943ddfa6cbd2dc6e409469180ad3a9de6ef3f2fe
Parents: 12fd0fa
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 30 15:13:57 2017 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Apr 5 18:36:44 2017 -0400
----------------------------------------------------------------------
.../end2end/SystemTablePermissionsIT.java | 263 +++++++++++++++++++
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +
.../query/ConnectionQueryServicesImpl.java | 51 +++-
.../query/ConnectionQueryServicesImplTest.java | 73 +++++
4 files changed, 380 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/943ddfa6/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
new file mode 100644
index 0000000..e99f322
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.security.access.AccessControlClient;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.phoenix.query.QueryServices;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test that verifies a user can read Phoenix tables with a minimal set of permissions.
+ */
+public class SystemTablePermissionsIT {
+ private static String SUPERUSER;
+
+ private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
+ "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
+ "SYSTEM.MUTEX"));
+ // PHOENIX-XXXX SYSTEM.MUTEX isn't being created in the SYSTEM namespace as it should be.
+ private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
+ Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
+ "SYSTEM.MUTEX"));
+
+ private static final String TABLE_NAME =
+ SystemTablePermissionsIT.class.getSimpleName().toUpperCase();
+ private static final int NUM_RECORDS = 5;
+
+ private HBaseTestingUtility testUtil = null;
+ private Properties clientProperties = null;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ SUPERUSER = System.getProperty("user.name");
+ }
+
+ private static void setCommonConfigProperties(Configuration conf) {
+ conf.set("hbase.coprocessor.master.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.coprocessor.region.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.coprocessor.regionserver.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.security.exec.permission.checks", "true");
+ conf.set("hbase.security.authorization", "true");
+ conf.set("hbase.superuser", SUPERUSER);
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ if (null != testUtil) {
+ testUtil.shutdownMiniCluster();
+ testUtil = null;
+ }
+ }
+
+ @Test
+ public void testSystemTablePermissions() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ clientProperties = new Properties();
+ Configuration conf = testUtil.getConfiguration();
+ setCommonConfigProperties(conf);
+ conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "false");
+ clientProperties.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "false");
+ testUtil.startMiniCluster(1);
+ final UserGroupInformation superUser = UserGroupInformation.createUserForTesting(
+ SUPERUSER, new String[0]);
+ final UserGroupInformation regularUser = UserGroupInformation.createUserForTesting(
+ "user", new String[0]);
+
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ createTable();
+ readTable();
+ return null;
+ }
+ });
+
+ Set<String> tables = getHBaseTables();
+ assertTrue("HBase tables do not include expected Phoenix tables: " + tables,
+ tables.containsAll(PHOENIX_SYSTEM_TABLES));
+
+ // Grant permission to the system tables for the unprivileged user
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ grantPermissions(regularUser.getShortUserName(), PHOENIX_SYSTEM_TABLES,
+ Action.EXEC, Action.READ);
+ grantPermissions(regularUser.getShortUserName(),
+ Collections.singleton(TABLE_NAME), Action.READ);
+ } catch (Throwable e) {
+ if (e instanceof Exception) {
+ throw (Exception) e;
+ } else {
+ throw new Exception(e);
+ }
+ }
+ return null;
+ }
+ });
+
+ // Make sure that the unprivileged user can read the table
+ regularUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // We expect this to not throw an error
+ readTable();
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testNamespaceMappedSystemTables() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ clientProperties = new Properties();
+ Configuration conf = testUtil.getConfiguration();
+ setCommonConfigProperties(conf);
+ testUtil.getConfiguration().set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ clientProperties.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ testUtil.startMiniCluster(1);
+ final UserGroupInformation superUser =
+ UserGroupInformation.createUserForTesting(SUPERUSER, new String[0]);
+ final UserGroupInformation regularUser =
+ UserGroupInformation.createUserForTesting("user", new String[0]);
+
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ createTable();
+ readTable();
+ return null;
+ }
+ });
+
+ Set<String> tables = getHBaseTables();
+ assertTrue("HBase tables do not include expected Phoenix tables: " + tables,
+ tables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
+
+ // Grant permission to the system tables for the unprivileged user
+ // An unprivileged user should only need to be able to Read and eXecute on them.
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ grantPermissions(regularUser.getShortUserName(),
+ PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, Action.EXEC, Action.READ);
+ grantPermissions(regularUser.getShortUserName(),
+ Collections.singleton(TABLE_NAME), Action.READ);
+ } catch (Throwable e) {
+ if (e instanceof Exception) {
+ throw (Exception) e;
+ } else {
+ throw new Exception(e);
+ }
+ }
+ return null;
+ }
+ });
+
+ regularUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // We expect this to not throw an error
+ readTable();
+ return null;
+ }
+ });
+ }
+
+ private String getJdbcUrl() {
+ return "jdbc:phoenix:localhost:" + testUtil.getZkCluster().getClientPort() + ":/hbase";
+ }
+
+ private void createTable() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProperties);
+ Statement stmt = conn.createStatement();) {
+ assertFalse(stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME));
+ assertFalse(stmt.execute("CREATE TABLE " + TABLE_NAME
+ + "(pk INTEGER not null primary key, data VARCHAR)"));
+ try (PreparedStatement pstmt = conn.prepareStatement("UPSERT INTO "
+ + TABLE_NAME + " values(?, ?)")) {
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ pstmt.setInt(1, i);
+ pstmt.setString(2, Integer.toString(i));
+ assertEquals(1, pstmt.executeUpdate());
+ }
+ }
+ conn.commit();
+ }
+ }
+
+ private void readTable() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProperties);
+ Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery("SELECT pk, data FROM " + TABLE_NAME);
+ assertNotNull(rs);
+ int i = 0;
+ while (rs.next()) {
+ assertEquals(i, rs.getInt(1));
+ assertEquals(Integer.toString(i), rs.getString(2));
+ i++;
+ }
+ assertEquals(NUM_RECORDS, i);
+ }
+ }
+
+ private void grantPermissions(String toUser, Set<String> tablesToGrant, Action... actions)
+ throws Throwable {
+ for (String table : tablesToGrant) {
+ AccessControlClient.grant(testUtil.getConfiguration(), TableName.valueOf(table), toUser,
+ null, null, actions);
+ }
+ }
+
+ private Set<String> getHBaseTables() throws IOException {
+ Set<String> tables = new HashSet<>();
+ for (TableName tn : testUtil.getHBaseAdmin().listTableNames()) {
+ tables.add(tn.getNameAsString());
+ }
+ return tables;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/943ddfa6/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index e09f5ba..195727b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -30,6 +30,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.ColumnProjector;
@@ -97,6 +98,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_CATALOG = SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"";
public static final String SYSTEM_CATALOG_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA,
SYSTEM_CATALOG_TABLE);
+ public static final TableName SYSTEM_CATALOG_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_CATALOG_NAME);
public static final byte[] SYSTEM_CATALOG_NAME_BYTES = Bytes.toBytes(SYSTEM_CATALOG_NAME);
public static final String SYSTEM_STATS_TABLE = "STATS";
public static final String SYSTEM_STATS_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_STATS_TABLE);
@@ -305,6 +307,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_MUTEX_TABLE_NAME = "MUTEX";
public static final String SYSTEM_MUTEX_NAME = SchemaUtil.getTableName(QueryConstants.SYSTEM_SCHEMA_NAME, SYSTEM_MUTEX_TABLE_NAME);
+ public static final TableName SYSTEM_MUTEX_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_MUTEX_NAME);
public static final byte[] SYSTEM_MUTEX_NAME_BYTES = Bytes.toBytes(SYSTEM_MUTEX_NAME);
public static final byte[] SYSTEM_MUTEX_FAMILY_NAME_BYTES = TABLE_FAMILY_BYTES;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/943ddfa6/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 9cd6a29..7d65d5a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -121,6 +121,7 @@ import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
import org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator;
+import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -253,6 +254,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -1015,7 +1017,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
QueryServicesOptions.DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE);
}
- private NamespaceDescriptor ensureNamespaceCreated(String schemaName) throws SQLException {
+ void ensureNamespaceCreated(String schemaName) throws SQLException {
SQLException sqlE = null;
try (HBaseAdmin admin = getAdmin()) {
NamespaceDescriptor namespaceDescriptor = null;
@@ -1028,13 +1030,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
namespaceDescriptor = NamespaceDescriptor.create(schemaName).build();
admin.createNamespace(namespaceDescriptor);
}
- return namespaceDescriptor;
+ return;
} catch (IOException e) {
sqlE = ServerUtil.parseServerException(e);
} finally {
if (sqlE != null) { throw sqlE; }
}
- return null; // will never make it here
}
/**
@@ -2441,6 +2442,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
ConnectionQueryServicesImpl.this.upgradeRequired.set(true);
}
+ } catch (PhoenixIOException e) {
+ if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
+ // Pass
+ logger.warn("Could not check for Phoenix SYSTEM tables, assuming they exist and are properly configured");
+ checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, getProps()).getName());
+ success = true;
+ } else {
+ initializationException = e;
+ }
+ return null;
}
if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
createOtherSystemTables(metaConnection);
@@ -2493,8 +2504,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void createSysMutexTable(HBaseAdmin admin) throws IOException, SQLException {
try {
- HTableDescriptor tableDesc = new HTableDescriptor(
- TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME_BYTES));
+ final String mutexTableName = PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME;
+ List<String> systemTables = getSystemTableNames(admin);
+ if (systemTables.contains(mutexTableName)) {
+ logger.debug("System mutex table already appears to exist, not creating it");
+ return;
+ }
+ HTableDescriptor tableDesc = new HTableDescriptor(mutexTableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(
PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES);
columnDesc.setTimeToLive(TTL_FOR_MUTEX); // Let mutex expire after some time
@@ -2512,6 +2528,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ List<String> getSystemTableNames(HBaseAdmin admin) throws IOException {
+ return Lists.newArrayList(admin.getTableNames(QueryConstants.SYSTEM_SCHEMA_NAME + "\\..*"));
+ }
+
private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQLException {
try {
metaConnection.createStatement().execute(QueryConstants.CREATE_SEQUENCE_METADATA);
@@ -3078,23 +3098,34 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
}
-
- private void ensureSystemTablesUpgraded(ReadOnlyProps props)
+
+ void ensureSystemTablesUpgraded(ReadOnlyProps props)
throws SQLException, IOException, IllegalArgumentException, InterruptedException {
if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, props)) { return; }
HTableInterface metatable = null;
try (HBaseAdmin admin = getAdmin()) {
- ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ // Namespace-mapping is enabled at this point.
+ try {
+ ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ } catch (PhoenixIOException e) {
+ // We could either:
+ // 1) Not access the NS descriptor. The NS may or may not exist at this point.
+ // 2) We could not create the NS
+ // Regardless of the case 1 or 2, if the NS does not exist, we will error expectedly
+ // below. If the NS does exist and is mapped, the below check will exit gracefully.
+ }
- List<String> tableNames = Lists.newArrayList(admin.getTableNames(QueryConstants.SYSTEM_SCHEMA_NAME + "\\..*"));
+ List<String> tableNames = getSystemTableNames(admin);
+ // No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
if (tableNames.size() == 0) { return; }
+ // Try to move any remaining tables matching "SYSTEM\..*" into "SYSTEM:"
if (tableNames.size() > 5) {
logger.warn("Expected 5 system tables but found " + tableNames.size() + ":" + tableNames);
}
byte[] mappedSystemTable = SchemaUtil
.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props).getName();
metatable = getTable(mappedSystemTable);
- if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)) {
+ if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)) {
if (!admin.tableExists(mappedSystemTable)) {
UpgradeUtil.mapTableToNamespace(admin, metatable,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props, null, PTableType.SYSTEM,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/943ddfa6/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
new file mode 100644
index 0000000..f7676d7
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Test;
+
+public class ConnectionQueryServicesImplTest {
+ private static final PhoenixIOException PHOENIX_IO_EXCEPTION = new PhoenixIOException(new Exception("Test exception"));
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testExceptionHandlingOnSystemNamespaceCreation() throws Exception {
+ ConnectionQueryServicesImpl cqs = mock(ConnectionQueryServicesImpl.class);
+ // Invoke the real methods for these two calls
+ when(cqs.createSchema(any(List.class), anyString())).thenCallRealMethod();
+ doCallRealMethod().when(cqs).ensureSystemTablesUpgraded(any(ReadOnlyProps.class));
+
+ // Spoof out this call so that ensureSystemTablesUpgrade() will return-fast.
+ when(cqs.getSystemTableNames(any(HBaseAdmin.class))).thenReturn(Collections.<String> emptyList());
+
+ // Throw a special exception to check on later
+ doThrow(PHOENIX_IO_EXCEPTION).when(cqs).ensureNamespaceCreated(anyString());
+
+ // Make sure that ensureSystemTablesUpgraded will try to migrate the system tables.
+ Map<String,String> props = new HashMap<>();
+ props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ cqs.ensureSystemTablesUpgraded(new ReadOnlyProps(props));
+
+ // Should be called after upgradeSystemTables()
+ // Proves that execution proceeded
+ verify(cqs).getSystemTableNames(any(HBaseAdmin.class));
+
+ try {
+ // Verifies that the exception is propagated back to the caller
+ cqs.createSchema(Collections.<Mutation> emptyList(), "");
+ } catch (PhoenixIOException e) {
+ assertEquals(PHOENIX_IO_EXCEPTION, e);
+ }
+ }
+}
[2/3] phoenix git commit: PHOENIX-3756 Handle users lacking ADMIN for
the SYSTEM namespace
Posted by el...@apache.org.
PHOENIX-3756 Handle users lacking ADMIN for the SYSTEM namespace
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1897fc45
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1897fc45
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1897fc45
Branch: refs/heads/4.x-HBase-1.1
Commit: 1897fc4586390c14fa0c7adba4209b5d089b6bae
Parents: f5f00b5
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 30 15:13:57 2017 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Apr 5 17:48:57 2017 -0400
----------------------------------------------------------------------
.../end2end/SystemTablePermissionsIT.java | 263 +++++++++++++++++++
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +
.../query/ConnectionQueryServicesImpl.java | 56 +++-
.../query/ConnectionQueryServicesImplTest.java | 73 +++++
4 files changed, 383 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1897fc45/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
new file mode 100644
index 0000000..9f213c8
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.security.access.AccessControlClient;
+import org.apache.hadoop.hbase.security.access.Permission.Action;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.phoenix.query.QueryServices;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test that verifies a user can read Phoenix tables with a minimal set of permissions.
+ */
+public class SystemTablePermissionsIT {
+ private static String SUPERUSER;
+
+ private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
+ "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
+ "SYSTEM.MUTEX"));
+ // PHOENIX-XXXX SYSTEM.MUTEX isn't being created in the SYSTEM namespace as it should be.
+ private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
+ Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
+ "SYSTEM.MUTEX"));
+
+ private static final String TABLE_NAME =
+ SystemTablePermissionsIT.class.getSimpleName().toUpperCase();
+ private static final int NUM_RECORDS = 5;
+
+ private HBaseTestingUtility testUtil = null;
+ private Properties clientProperties = null;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ SUPERUSER = System.getProperty("user.name");
+ }
+
+ private static void setCommonConfigProperties(Configuration conf) {
+ conf.set("hbase.coprocessor.master.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.coprocessor.region.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.coprocessor.regionserver.classes",
+ "org.apache.hadoop.hbase.security.access.AccessController");
+ conf.set("hbase.security.exec.permission.checks", "true");
+ conf.set("hbase.security.authorization", "true");
+ conf.set("hbase.superuser", SUPERUSER);
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ if (null != testUtil) {
+ testUtil.shutdownMiniCluster();
+ testUtil = null;
+ }
+ }
+
+ @Test
+ public void testSystemTablePermissions() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ clientProperties = new Properties();
+ Configuration conf = testUtil.getConfiguration();
+ setCommonConfigProperties(conf);
+ conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "false");
+ clientProperties.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "false");
+ testUtil.startMiniCluster(1);
+ final UserGroupInformation superUser = UserGroupInformation.createUserForTesting(
+ SUPERUSER, new String[0]);
+ final UserGroupInformation regularUser = UserGroupInformation.createUserForTesting(
+ "user", new String[0]);
+
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ createTable();
+ readTable();
+ return null;
+ }
+ });
+
+ Set<String> tables = getHBaseTables();
+ assertTrue("HBase tables do not include expected Phoenix tables: " + tables,
+ tables.containsAll(PHOENIX_SYSTEM_TABLES));
+
+ // Grant permission to the system tables for the unprivileged user
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ grantPermissions(regularUser.getShortUserName(), PHOENIX_SYSTEM_TABLES,
+ Action.EXEC, Action.READ);
+ grantPermissions(regularUser.getShortUserName(),
+ Collections.singleton(TABLE_NAME), Action.READ);
+ } catch (Throwable e) {
+ if (e instanceof Exception) {
+ throw (Exception) e;
+ } else {
+ throw new Exception(e);
+ }
+ }
+ return null;
+ }
+ });
+
+ // Make sure that the unprivileged user can read the table
+ regularUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // We expect this to not throw an error
+ readTable();
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testNamespaceMappedSystemTables() throws Exception {
+ testUtil = new HBaseTestingUtility();
+ clientProperties = new Properties();
+ Configuration conf = testUtil.getConfiguration();
+ setCommonConfigProperties(conf);
+ testUtil.getConfiguration().set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ clientProperties.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ testUtil.startMiniCluster(1);
+ final UserGroupInformation superUser =
+ UserGroupInformation.createUserForTesting(SUPERUSER, new String[0]);
+ final UserGroupInformation regularUser =
+ UserGroupInformation.createUserForTesting("user", new String[0]);
+
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ createTable();
+ readTable();
+ return null;
+ }
+ });
+
+ Set<String> tables = getHBaseTables();
+ assertTrue("HBase tables do not include expected Phoenix tables: " + tables,
+ tables.containsAll(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES));
+
+ // Grant permission to the system tables for the unprivileged user
+ // An unprivileged user should only need to be able to Read and eXecute on them.
+ superUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ grantPermissions(regularUser.getShortUserName(),
+ PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, Action.EXEC, Action.READ);
+ grantPermissions(regularUser.getShortUserName(),
+ Collections.singleton(TABLE_NAME), Action.READ);
+ } catch (Throwable e) {
+ if (e instanceof Exception) {
+ throw (Exception) e;
+ } else {
+ throw new Exception(e);
+ }
+ }
+ return null;
+ }
+ });
+
+ regularUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // We expect this to not throw an error
+ readTable();
+ return null;
+ }
+ });
+ }
+
+ private String getJdbcUrl() {
+ return "jdbc:phoenix:localhost:" + testUtil.getZkCluster().getClientPort() + ":/hbase";
+ }
+
+ private void createTable() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProperties);
+ Statement stmt = conn.createStatement();) {
+ assertFalse(stmt.execute("DROP TABLE IF EXISTS " + TABLE_NAME));
+ assertFalse(stmt.execute("CREATE TABLE " + TABLE_NAME
+ + "(pk INTEGER not null primary key, data VARCHAR)"));
+ try (PreparedStatement pstmt = conn.prepareStatement("UPSERT INTO "
+ + TABLE_NAME + " values(?, ?)")) {
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ pstmt.setInt(1, i);
+ pstmt.setString(2, Integer.toString(i));
+ assertEquals(1, pstmt.executeUpdate());
+ }
+ }
+ conn.commit();
+ }
+ }
+
+ private void readTable() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(getJdbcUrl(), clientProperties);
+ Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery("SELECT pk, data FROM " + TABLE_NAME);
+ assertNotNull(rs);
+ int i = 0;
+ while (rs.next()) {
+ assertEquals(i, rs.getInt(1));
+ assertEquals(Integer.toString(i), rs.getString(2));
+ i++;
+ }
+ assertEquals(NUM_RECORDS, i);
+ }
+ }
+
+ private void grantPermissions(String toUser, Set<String> tablesToGrant, Action... actions)
+ throws Throwable {
+ for (String table : tablesToGrant) {
+ AccessControlClient.grant(testUtil.getConnection(), TableName.valueOf(table), toUser,
+ null, null, actions);
+ }
+ }
+
+ private Set<String> getHBaseTables() throws IOException {
+ Set<String> tables = new HashSet<>();
+ for (TableName tn : testUtil.getHBaseAdmin().listTableNames()) {
+ tables.add(tn.getNameAsString());
+ }
+ return tables;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1897fc45/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index e3a206c..e061406 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -30,6 +30,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.ColumnProjector;
@@ -97,6 +98,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_CATALOG = SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"";
public static final String SYSTEM_CATALOG_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA,
SYSTEM_CATALOG_TABLE);
+ public static final TableName SYSTEM_CATALOG_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_CATALOG_NAME);
public static final byte[] SYSTEM_CATALOG_NAME_BYTES = Bytes.toBytes(SYSTEM_CATALOG_NAME);
public static final String SYSTEM_STATS_TABLE = "STATS";
public static final String SYSTEM_STATS_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_STATS_TABLE);
@@ -305,6 +307,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_MUTEX_TABLE_NAME = "MUTEX";
public static final String SYSTEM_MUTEX_NAME = SchemaUtil.getTableName(QueryConstants.SYSTEM_SCHEMA_NAME, SYSTEM_MUTEX_TABLE_NAME);
+ public static final TableName SYSTEM_MUTEX_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_MUTEX_NAME);
public static final byte[] SYSTEM_MUTEX_NAME_BYTES = Bytes.toBytes(SYSTEM_MUTEX_NAME);
public static final byte[] SYSTEM_MUTEX_FAMILY_NAME_BYTES = TABLE_FAMILY_BYTES;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1897fc45/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 1d0dd9f..0ca5995 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -121,6 +121,7 @@ import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
import org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator;
+import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -253,6 +254,7 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -1014,7 +1016,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
QueryServicesOptions.DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE);
}
- private NamespaceDescriptor ensureNamespaceCreated(String schemaName) throws SQLException {
+ void ensureNamespaceCreated(String schemaName) throws SQLException {
SQLException sqlE = null;
try (HBaseAdmin admin = getAdmin()) {
NamespaceDescriptor namespaceDescriptor = null;
@@ -1027,13 +1029,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
namespaceDescriptor = NamespaceDescriptor.create(schemaName).build();
admin.createNamespace(namespaceDescriptor);
}
- return namespaceDescriptor;
+ return;
} catch (IOException e) {
sqlE = ServerUtil.parseServerException(e);
} finally {
if (sqlE != null) { throw sqlE; }
}
- return null; // will never make it here
}
/**
@@ -2445,6 +2446,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
ConnectionQueryServicesImpl.this.upgradeRequired.set(true);
}
+ } catch (PhoenixIOException e) {
+ if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
+ // Pass
+ logger.warn("Could not check for Phoenix SYSTEM tables, assuming they exist and are properly configured");
+ checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, getProps()).getName());
+ success = true;
+ } else {
+ initializationException = e;
+ }
+ return null;
}
if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
createOtherSystemTables(metaConnection);
@@ -2497,8 +2508,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void createSysMutexTable(HBaseAdmin admin) throws IOException, SQLException {
try {
- HTableDescriptor tableDesc = new HTableDescriptor(
- TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME_BYTES));
+ final TableName mutexTableName = TableName.valueOf(
+ PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME_BYTES);
+ List<TableName> systemTables = getSystemTableNames(admin);
+ if (systemTables.contains(mutexTableName)) {
+ logger.debug("System mutex table already appears to exist, not creating it");
+ return;
+ }
+ HTableDescriptor tableDesc = new HTableDescriptor(mutexTableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(
PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES);
columnDesc.setTimeToLive(TTL_FOR_MUTEX); // Let mutex expire after some time
@@ -2516,6 +2533,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ List<TableName> getSystemTableNames(HBaseAdmin admin) throws IOException {
+ return Lists.newArrayList(admin.listTableNames(QueryConstants.SYSTEM_SCHEMA_NAME + "\\..*"));
+ }
+
private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQLException {
try {
metaConnection.createStatement().execute(QueryConstants.CREATE_SEQUENCE_METADATA);
@@ -3081,23 +3102,34 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
}
-
- private void ensureSystemTablesUpgraded(ReadOnlyProps props)
+
+ void ensureSystemTablesUpgraded(ReadOnlyProps props)
throws SQLException, IOException, IllegalArgumentException, InterruptedException {
if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, props)) { return; }
HTableInterface metatable = null;
try (HBaseAdmin admin = getAdmin()) {
- ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ // Namespace-mapping is enabled at this point.
+ try {
+ ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ } catch (PhoenixIOException e) {
+ // We could either:
+ // 1) Not access the NS descriptor. The NS may or may not exist at this point.
+ // 2) We could not create the NS
+ // Regardless of the case 1 or 2, if the NS does not exist, we will error expectedly
+ // below. If the NS does exist and is mapped, the below check will exit gracefully.
+ }
- List<TableName> tableNames = Lists.newArrayList(admin.listTableNames(QueryConstants.SYSTEM_SCHEMA_NAME + "\\..*"));
+ List<TableName> tableNames = getSystemTableNames(admin);
+ // No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
if (tableNames.size() == 0) { return; }
+ // Try to move any remaining tables matching "SYSTEM\..*" into "SYSTEM:"
if (tableNames.size() > 5) {
logger.warn("Expected 5 system tables but found " + tableNames.size() + ":" + tableNames);
}
byte[] mappedSystemTable = SchemaUtil
.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props).getName();
metatable = getTable(mappedSystemTable);
- if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)) {
+ if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)) {
if (!admin.tableExists(mappedSystemTable)) {
UpgradeUtil.mapTableToNamespace(admin, metatable,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props, null, PTableType.SYSTEM,
@@ -3106,9 +3138,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0);
}
- tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
+ tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME);
}
- tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME);
+ tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_MUTEX_HBASE_TABLE_NAME);
for (TableName table : tableNames) {
UpgradeUtil.mapTableToNamespace(admin, metatable, table.getNameAsString(), props, null, PTableType.SYSTEM,
null);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1897fc45/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
new file mode 100644
index 0000000..73ddd2d
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.phoenix.exception.PhoenixIOException;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Test;
+
+public class ConnectionQueryServicesImplTest {
+ private static final PhoenixIOException PHOENIX_IO_EXCEPTION = new PhoenixIOException(new Exception("Test exception"));
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testExceptionHandlingOnSystemNamespaceCreation() throws Exception {
+ ConnectionQueryServicesImpl cqs = mock(ConnectionQueryServicesImpl.class);
+ // Invoke the real methods for these two calls
+ when(cqs.createSchema(any(List.class), anyString())).thenCallRealMethod();
+ doCallRealMethod().when(cqs).ensureSystemTablesUpgraded(any(ReadOnlyProps.class));
+
+ // Spoof out this call so that ensureSystemTablesUpgrade() will return-fast.
+ when(cqs.getSystemTableNames(any(HBaseAdmin.class))).thenReturn(Collections.<TableName> emptyList());
+
+ // Throw a special exception to check on later
+ doThrow(PHOENIX_IO_EXCEPTION).when(cqs).ensureNamespaceCreated(anyString());
+
+ // Make sure that ensureSystemTablesUpgraded will try to migrate the system tables.
+ Map<String,String> props = new HashMap<>();
+ props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
+ cqs.ensureSystemTablesUpgraded(new ReadOnlyProps(props));
+
+ // Should be called after upgradeSystemTables()
+ // Proves that execution proceeded
+ verify(cqs).getSystemTableNames(any(HBaseAdmin.class));
+
+ try {
+ // Verifies that the exception is propagated back to the caller
+ cqs.createSchema(Collections.<Mutation> emptyList(), "");
+ } catch (PhoenixIOException e) {
+ assertEquals(PHOENIX_IO_EXCEPTION, e);
+ }
+ }
+}