You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2017/09/27 00:15:25 UTC
[2/2] phoenix git commit: PHOENIX-4138 Create a hard limit on number
of indexes per table (Churro Morales)
PHOENIX-4138 Create a hard limit on number of indexes per table (Churro Morales)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4969794b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4969794b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4969794b
Branch: refs/heads/master
Commit: 4969794bf0c91305d01c8f016abe95039642d46e
Parents: 5d95727
Author: James Taylor <ja...@apache.org>
Authored: Tue Sep 26 16:42:31 2017 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Tue Sep 26 17:14:55 2017 -0700
----------------------------------------------------------------------
.../coprocessor/MetaDataEndpointImplTest.java | 44 ++++++++++++++++++++
.../apache/phoenix/end2end/CreateTableIT.java | 31 ++++++++++++++
.../coprocessor/MetaDataEndpointImpl.java | 15 +++++++
.../org/apache/phoenix/query/QueryServices.java | 4 +-
.../phoenix/query/QueryServicesOptions.java | 1 +
5 files changed, 94 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java
new file mode 100644
index 0000000..2c558d8
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MetaDataEndpointImplTest {
+
+ @Test
+ public void testExceededIndexQuota() throws Exception {
+ PTable parentTable = mock(PTable.class);
+ List<PTable> indexes = Lists.newArrayList(mock(PTable.class), mock(PTable.class));
+ when(parentTable.getIndexes()).thenReturn(indexes);
+ Configuration configuration = new Configuration();
+ assertFalse(MetaDataEndpointImpl.execeededIndexQuota(PTableType.INDEX, parentTable, configuration));
+ configuration.setInt(QueryServices.MAX_INDEXES_PER_TABLE, 1);
+ assertTrue(MetaDataEndpointImpl.execeededIndexQuota(PTableType.INDEX, parentTable, configuration));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 32d72f7..93bb02b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -42,6 +42,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
@@ -189,6 +190,36 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(86400, columnFamilies[0].getTimeToLive());
}
+ @Test
+ public void testCreatingTooManyIndexesIsNotAllowed() throws Exception {
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName + " (\n"
+ + "ID VARCHAR(15) PRIMARY KEY,\n"
+ + "COL1 BIGINT,"
+ + "COL2 BIGINT,"
+ + "COL3 BIGINT,"
+ + "COL4 BIGINT) ";
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(ddl);
+
+ int maxIndexes = conn.unwrap(PhoenixConnection.class).getQueryServices().getProps().getInt(
+ QueryServices.MAX_INDEXES_PER_TABLE, QueryServicesOptions.DEFAULT_MAX_INDEXES_PER_TABLE);
+
+ // Use local indexes since there's only one physical table for all of them.
+ for (int i = 0; i < maxIndexes; i++) {
+ conn.createStatement().execute("CREATE LOCAL INDEX I_" + i + tableName + " ON " + tableName + "(COL1) INCLUDE (COL2,COL3,COL4)");
+ }
+
+ // here we ensure we get a too many indexes error
+ try {
+ conn.createStatement().execute("CREATE LOCAL INDEX I_" + maxIndexes + tableName + " ON " + tableName + "(COL1) INCLUDE (COL2,COL3,COL4)");
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TOO_MANY_INDEXES.getErrorCode(), e.getErrorCode());
+ }
+ }
+
/**
* Tests that when: 1) DDL has both pk as well as key value columns 2) Key value columns have
* different column family names 3) TTL specifier doesn't have column family name. Then: 1)TTL
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 80b0785..6ae5af1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -96,6 +96,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -1432,6 +1433,13 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
done.run(builder.build());
return;
}
+ // make sure we haven't gone over our threshold for indexes on this table.
+ if (execeededIndexQuota(tableType, parentTable, env.getConfiguration())) {
+ builder.setReturnCode(MetaDataProtos.MutationCode.TOO_MANY_INDEXES);
+ builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis());
+ done.run(builder.build());
+ return;
+ }
long parentTableSeqNumber;
if (tableType == PTableType.VIEW && viewPhysicalTableRow != null && request.hasClientVersion()) {
// Starting 4.5, the client passes the sequence number of the physical table in the table metadata.
@@ -1633,6 +1641,13 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
}
+ @VisibleForTesting
+ static boolean execeededIndexQuota(PTableType tableType, PTable parentTable, Configuration configuration) {
+ return PTableType.INDEX == tableType && parentTable.getIndexes().size() >= configuration
+ .getInt(QueryServices.MAX_INDEXES_PER_TABLE,
+ QueryServicesOptions.DEFAULT_MAX_INDEXES_PER_TABLE);
+ }
+
private static RowLock acquireLock(Region region, byte[] key, List<RowLock> locks)
throws IOException {
RowLock rowLock = region.getRowLock(key, false);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 70d9878..a4a4124 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -262,7 +262,9 @@ public interface QueryServices extends SQLCloseable {
public static final String UPLOAD_BINARY_DATA_TYPE_ENCODING = "phoenix.upload.binaryDataType.encoding";
public static final String INDEX_ASYNC_BUILD_ENABLED = "phoenix.index.async.build.enabled";
-
+
+ public static final String MAX_INDEXES_PER_TABLE = "phoenix.index.maxIndexesPerTable";
+
public static final String CLIENT_CACHE_ENCODING = "phoenix.table.client.cache.encoding";
public static final String AUTO_UPGRADE_ENABLED = "phoenix.autoupgrade.enabled";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 94109aa..3f70b0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -145,6 +145,7 @@ public class QueryServicesOptions {
public static final int DEFAULT_TRACING_THREAD_POOL_SIZE = 5;
public static final int DEFAULT_TRACING_BATCH_SIZE = 100;
public static final int DEFAULT_TRACING_TRACE_BUFFER_SIZE = 1000;
+ public static final int DEFAULT_MAX_INDEXES_PER_TABLE = 10;
public final static int DEFAULT_MUTATE_BATCH_SIZE = 100; // Batch size for UPSERT SELECT and DELETE
//Batch size in bytes for UPSERT, SELECT and DELETE. By default, 2MB