You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2017/09/27 00:15:24 UTC
[1/2] phoenix git commit: PHOENIX-4239 Fix flapping test in
PartialIndexRebuilderIT
Repository: phoenix
Updated Branches:
refs/heads/master 5d9572736 -> 176f541ce
PHOENIX-4239 Fix flapping test in PartialIndexRebuilderIT
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/176f541c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/176f541c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/176f541c
Branch: refs/heads/master
Commit: 176f541ceb36c74ecdb88d113132a4ff2e44a86b
Parents: 4969794
Author: James Taylor <ja...@apache.org>
Authored: Tue Sep 26 15:35:36 2017 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Tue Sep 26 17:14:55 2017 -0700
----------------------------------------------------------------------
.../end2end/index/PartialIndexRebuilderIT.java | 8 +--
.../java/org/apache/phoenix/util/TestUtil.java | 51 ++++++++++++++++----
2 files changed, 46 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/176f541c/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
index 9095dbe..8bf2bc8 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
@@ -88,7 +88,7 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT {
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString());
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, Long.toString(REBUILD_INTERVAL));
- serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "300000"); // give up rebuilding after 5 minutes
+ serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "50000000");
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_PERIOD, Long.toString(REBUILD_PERIOD)); // batch at 50 seconds
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(WAIT_AFTER_DISABLED));
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), ReadOnlyProps.EMPTY_PROPS);
@@ -897,7 +897,7 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT {
TestUtil.removeCoprocessor(conn, fullIndexName, WriteFailingRegionObserver.class);
runIndexRebuilder();
- assertTrue(TestUtil.checkIndexState(conn, fullIndexName, indexStateOnFailure == PIndexState.DISABLE ? PIndexState.INACTIVE : PIndexState.ACTIVE, null));
+ assertEquals(indexStateOnFailure == PIndexState.DISABLE ? PIndexState.INACTIVE : PIndexState.ACTIVE, TestUtil.getIndexState(conn, fullIndexName));
clock.time += WAIT_AFTER_DISABLED;
// First batch should have been processed again because we started over
@@ -907,7 +907,9 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT {
clock.time += 2 * REBUILD_PERIOD;
// Second batch should have been processed now
runIndexRebuilder();
- assertTrue(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.ACTIVE, 0L));
+ clock.time += 2 * REBUILD_PERIOD;
+ runIndexRebuilder();
+ TestUtil.assertIndexState(conn, fullIndexName, PIndexState.ACTIVE, 0L);
// Verify that other batches were processed
IndexScrutiny.scrutinizeIndex(conn, fullTableName, fullIndexName);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/176f541c/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 45fd52c..8b93b5c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -888,14 +888,25 @@ public class TestUtil {
waitForIndexState(conn, fullIndexName, indexState, 0L);
}
- private enum IndexStateCheck {SUCCESS, FAIL, KEEP_TRYING};
+ private static class IndexStateCheck {
+ public final PIndexState indexState;
+ public final Long indexDisableTimestamp;
+ public final Boolean success;
+
+ public IndexStateCheck(PIndexState indexState, Long indexDisableTimestamp, Boolean success) {
+ this.indexState = indexState;
+ this.indexDisableTimestamp = indexDisableTimestamp;
+ this.success = success;
+ }
+ }
+
public static void waitForIndexState(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws InterruptedException, SQLException {
int maxTries = 60, nTries = 0;
do {
Thread.sleep(1000); // sleep 1 sec
IndexStateCheck state = checkIndexStateInternal(conn, fullIndexName, expectedIndexState, expectedIndexDisableTimestamp);
- if (state != IndexStateCheck.KEEP_TRYING) {
- if (state == IndexStateCheck.SUCCESS) {
+ if (state.success != null) {
+ if (Boolean.TRUE.equals(state.success)) {
return;
}
fail("Index state will not become " + expectedIndexState);
@@ -905,8 +916,26 @@ public class TestUtil {
}
public static boolean checkIndexState(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws SQLException {
- return checkIndexStateInternal(conn,fullIndexName, expectedIndexState, expectedIndexDisableTimestamp) == IndexStateCheck.SUCCESS;
+ return Boolean.TRUE.equals(checkIndexStateInternal(conn,fullIndexName, expectedIndexState, expectedIndexDisableTimestamp).success);
+ }
+
+ public static void assertIndexState(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws SQLException {
+ IndexStateCheck state = checkIndexStateInternal(conn,fullIndexName, expectedIndexState, expectedIndexDisableTimestamp);
+ if (!Boolean.TRUE.equals(state.success)) {
+ if (expectedIndexState != null) {
+ assertEquals(expectedIndexState, state.indexState);
+ }
+ if (expectedIndexDisableTimestamp != null) {
+ assertEquals(expectedIndexDisableTimestamp, state.indexDisableTimestamp);
+ }
+ }
+ }
+
+ public static PIndexState getIndexState(Connection conn, String fullIndexName) throws SQLException {
+ IndexStateCheck state = checkIndexStateInternal(conn, fullIndexName, null, null);
+ return state.indexState;
}
+
private static IndexStateCheck checkIndexStateInternal(Connection conn, String fullIndexName, PIndexState expectedIndexState, Long expectedIndexDisableTimestamp) throws SQLException {
String schema = SchemaUtil.getSchemaNameFromFullName(fullIndexName);
String index = SchemaUtil.getTableNameFromFullName(fullIndexName);
@@ -915,19 +944,21 @@ public class TestUtil {
+ ") = (" + "'" + schema + "','" + index + "') "
+ "AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NULL AND " + PhoenixDatabaseMetaData.COLUMN_NAME + " IS NULL";
ResultSet rs = conn.createStatement().executeQuery(query);
+ Long actualIndexDisableTimestamp = null;
+ PIndexState actualIndexState = null;
if (rs.next()) {
- Long actualIndexDisableTimestamp = rs.getLong(1);
- PIndexState actualIndexState = PIndexState.fromSerializedValue(rs.getString(2));
+ actualIndexDisableTimestamp = rs.getLong(1);
+ actualIndexState = PIndexState.fromSerializedValue(rs.getString(2));
boolean matchesExpected = (expectedIndexDisableTimestamp == null || Objects.equal(actualIndexDisableTimestamp, expectedIndexDisableTimestamp))
- && actualIndexState == expectedIndexState;
+ && (expectedIndexState == null || actualIndexState == expectedIndexState);
if (matchesExpected) {
- return IndexStateCheck.SUCCESS;
+ return new IndexStateCheck(actualIndexState, actualIndexDisableTimestamp, Boolean.TRUE);
}
if (ZERO.equals(actualIndexDisableTimestamp)) {
- return IndexStateCheck.FAIL;
+ return new IndexStateCheck(actualIndexState, actualIndexDisableTimestamp, Boolean.FALSE);
}
}
- return IndexStateCheck.KEEP_TRYING;
+ return new IndexStateCheck(actualIndexState, actualIndexDisableTimestamp, null);
}
public static long getRowCount(Connection conn, String tableName) throws SQLException {
[2/2] phoenix git commit: PHOENIX-4138 Create a hard limit on number
of indexes per table (Churro Morales)
Posted by ja...@apache.org.
PHOENIX-4138 Create a hard limit on number of indexes per table (Churro Morales)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4969794b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4969794b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4969794b
Branch: refs/heads/master
Commit: 4969794bf0c91305d01c8f016abe95039642d46e
Parents: 5d95727
Author: James Taylor <ja...@apache.org>
Authored: Tue Sep 26 16:42:31 2017 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Tue Sep 26 17:14:55 2017 -0700
----------------------------------------------------------------------
.../coprocessor/MetaDataEndpointImplTest.java | 44 ++++++++++++++++++++
.../apache/phoenix/end2end/CreateTableIT.java | 31 ++++++++++++++
.../coprocessor/MetaDataEndpointImpl.java | 15 +++++++
.../org/apache/phoenix/query/QueryServices.java | 4 +-
.../phoenix/query/QueryServicesOptions.java | 1 +
5 files changed, 94 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java
new file mode 100644
index 0000000..2c558d8
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/MetaDataEndpointImplTest.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MetaDataEndpointImplTest {
+
+ @Test
+ public void testExceededIndexQuota() throws Exception {
+ PTable parentTable = mock(PTable.class);
+ List<PTable> indexes = Lists.newArrayList(mock(PTable.class), mock(PTable.class));
+ when(parentTable.getIndexes()).thenReturn(indexes);
+ Configuration configuration = new Configuration();
+ assertFalse(MetaDataEndpointImpl.execeededIndexQuota(PTableType.INDEX, parentTable, configuration));
+ configuration.setInt(QueryServices.MAX_INDEXES_PER_TABLE, 1);
+ assertTrue(MetaDataEndpointImpl.execeededIndexQuota(PTableType.INDEX, parentTable, configuration));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 32d72f7..93bb02b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -42,6 +42,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
@@ -189,6 +190,36 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(86400, columnFamilies[0].getTimeToLive());
}
+ @Test
+ public void testCreatingTooManyIndexesIsNotAllowed() throws Exception {
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName + " (\n"
+ + "ID VARCHAR(15) PRIMARY KEY,\n"
+ + "COL1 BIGINT,"
+ + "COL2 BIGINT,"
+ + "COL3 BIGINT,"
+ + "COL4 BIGINT) ";
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute(ddl);
+
+ int maxIndexes = conn.unwrap(PhoenixConnection.class).getQueryServices().getProps().getInt(
+ QueryServices.MAX_INDEXES_PER_TABLE, QueryServicesOptions.DEFAULT_MAX_INDEXES_PER_TABLE);
+
+ // Use local indexes since there's only one physical table for all of them.
+ for (int i = 0; i < maxIndexes; i++) {
+ conn.createStatement().execute("CREATE LOCAL INDEX I_" + i + tableName + " ON " + tableName + "(COL1) INCLUDE (COL2,COL3,COL4)");
+ }
+
+ // here we ensure we get a too many indexes error
+ try {
+ conn.createStatement().execute("CREATE LOCAL INDEX I_" + maxIndexes + tableName + " ON " + tableName + "(COL1) INCLUDE (COL2,COL3,COL4)");
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TOO_MANY_INDEXES.getErrorCode(), e.getErrorCode());
+ }
+ }
+
/**
* Tests that when: 1) DDL has both pk as well as key value columns 2) Key value columns have
* different column family names 3) TTL specifier doesn't have column family name. Then: 1)TTL
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 80b0785..6ae5af1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -96,6 +96,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -1432,6 +1433,13 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
done.run(builder.build());
return;
}
+ // make sure we haven't gone over our threshold for indexes on this table.
+ if (execeededIndexQuota(tableType, parentTable, env.getConfiguration())) {
+ builder.setReturnCode(MetaDataProtos.MutationCode.TOO_MANY_INDEXES);
+ builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis());
+ done.run(builder.build());
+ return;
+ }
long parentTableSeqNumber;
if (tableType == PTableType.VIEW && viewPhysicalTableRow != null && request.hasClientVersion()) {
// Starting 4.5, the client passes the sequence number of the physical table in the table metadata.
@@ -1633,6 +1641,13 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
}
+ @VisibleForTesting
+ static boolean execeededIndexQuota(PTableType tableType, PTable parentTable, Configuration configuration) {
+ return PTableType.INDEX == tableType && parentTable.getIndexes().size() >= configuration
+ .getInt(QueryServices.MAX_INDEXES_PER_TABLE,
+ QueryServicesOptions.DEFAULT_MAX_INDEXES_PER_TABLE);
+ }
+
private static RowLock acquireLock(Region region, byte[] key, List<RowLock> locks)
throws IOException {
RowLock rowLock = region.getRowLock(key, false);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 70d9878..a4a4124 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -262,7 +262,9 @@ public interface QueryServices extends SQLCloseable {
public static final String UPLOAD_BINARY_DATA_TYPE_ENCODING = "phoenix.upload.binaryDataType.encoding";
public static final String INDEX_ASYNC_BUILD_ENABLED = "phoenix.index.async.build.enabled";
-
+
+ public static final String MAX_INDEXES_PER_TABLE = "phoenix.index.maxIndexesPerTable";
+
public static final String CLIENT_CACHE_ENCODING = "phoenix.table.client.cache.encoding";
public static final String AUTO_UPGRADE_ENABLED = "phoenix.autoupgrade.enabled";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4969794b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 94109aa..3f70b0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -145,6 +145,7 @@ public class QueryServicesOptions {
public static final int DEFAULT_TRACING_THREAD_POOL_SIZE = 5;
public static final int DEFAULT_TRACING_BATCH_SIZE = 100;
public static final int DEFAULT_TRACING_TRACE_BUFFER_SIZE = 1000;
+ public static final int DEFAULT_MAX_INDEXES_PER_TABLE = 10;
public final static int DEFAULT_MUTATE_BATCH_SIZE = 100; // Batch size for UPSERT SELECT and DELETE
//Batch size in bytes for UPSERT, SELECT and DELETE. By default, 2MB