You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2017/08/25 20:57:22 UTC
[3/3] phoenix git commit: PHOENIX-3953 Clear INDEX_DISABLED_TIMESTAMP
and disable index on compaction
PHOENIX-3953 Clear INDEX_DISABLED_TIMESTAMP and disable index on compaction
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b8595f1a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b8595f1a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b8595f1a
Branch: refs/heads/4.x-HBase-1.1
Commit: b8595f1afcaedcf6e51150597b6ae2261d2856a9
Parents: 9f2c09c
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Aug 25 11:38:41 2017 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Aug 25 13:53:23 2017 -0700
----------------------------------------------------------------------
.../end2end/index/PartialIndexRebuilderIT.java | 32 ++++++++++++++
.../org/apache/phoenix/hbase/index/Indexer.java | 45 ++++++++++++++++++++
.../java/org/apache/phoenix/util/IndexUtil.java | 7 ++-
3 files changed, 83 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8595f1a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
index f42d2c9..9483e87 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
@@ -245,6 +245,38 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT {
}
@Test
+ public void testCompactionDuringRebuild() throws Throwable {
+ String schemaName = generateUniqueName();
+ String tableName = generateUniqueName();
+ String indexName = generateUniqueName();
+ final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+ String fullIndexName = SchemaUtil.getTableName(schemaName, indexName);
+ final MyClock clock = new MyClock(1000);
+ // Use our own clock to prevent race between partial rebuilder and compaction
+ EnvironmentEdgeManager.injectEdge(clock);
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute("CREATE TABLE " + fullTableName + "(k INTEGER PRIMARY KEY, v1 INTEGER, v2 INTEGER) COLUMN_ENCODED_BYTES = 0, STORE_NULLS=true, GUIDE_POSTS_WIDTH=1000");
+ clock.time += 1000;
+ conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullTableName + " (v1) INCLUDE (v2)");
+ clock.time += 1000;
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(1, 2, 3)");
+ conn.commit();
+ clock.time += 1000;
+ long disableTS = EnvironmentEdgeManager.currentTimeMillis();
+ HTableInterface metaTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
+ IndexUtil.updateIndexState(fullIndexName, disableTS, metaTable, PIndexState.DISABLE);
+ TestUtil.doMajorCompaction(conn, fullIndexName);
+ assertFalse(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE, 0L));
+ TestUtil.analyzeTable(conn, fullTableName);
+ assertFalse(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE, 0L));
+ TestUtil.doMajorCompaction(conn, fullTableName);
+ assertTrue(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE, 0L));
+ } finally {
+ EnvironmentEdgeManager.injectEdge(null);
+ }
+ }
+
+ @Test
@Repeat(5)
public void testDeleteAndUpsertAfterFailure() throws Throwable {
final int nRows = 10;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8595f1a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index 2ad50ec..8072fba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -25,6 +25,7 @@ import static org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITER
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -86,12 +87,18 @@ import org.apache.phoenix.hbase.index.write.IndexWriter;
import org.apache.phoenix.hbase.index.write.RecoveryIndexWriter;
import org.apache.phoenix.hbase.index.write.recovery.PerRegionIndexWriteCache;
import org.apache.phoenix.hbase.index.write.recovery.StoreFailuresInCachePolicy;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PIndexState;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.trace.TracingUtils;
import org.apache.phoenix.trace.util.NullSpan;
import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ServerUtil;
import com.google.common.collect.Lists;
@@ -836,5 +843,43 @@ public class Indexer extends BaseRegionObserver {
properties.put(Indexer.INDEX_BUILDER_CONF_KEY, builder.getName());
desc.addCoprocessor(Indexer.class.getName(), null, priority, properties);
}
+
+ @Override
+ public InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+ final InternalScanner scanner, final ScanType scanType) throws IOException {
+ // Compaction and split upcalls run with the effective user context of the requesting user.
+ // This will lead to failure of cross cluster RPC if the effective user is not
+ // the login user. Switch to the login user context to ensure we have the expected
+ // security context.
+ return User.runAsLoginUser(new PrivilegedExceptionAction<InternalScanner>() {
+ @Override
+ public InternalScanner run() throws Exception {
+ InternalScanner internalScanner = scanner;
+ if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
+ String fullTableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
+ try {
+ PhoenixConnection conn = QueryUtil.getConnectionOnServer(c.getEnvironment().getConfiguration()).unwrap(PhoenixConnection.class);
+ PTable table = PhoenixRuntime.getTableNoCache(conn, fullTableName);
+ for (PTable index : table.getIndexes()) {
+ if (index.getIndexDisableTimestamp() != 0) {
+ try {
+ IndexUtil.updateIndexState(conn, index.getName().getString(), PIndexState.DISABLE, Long.valueOf(0L));
+ } catch (SQLException e) {
+ LOG.warn("Unable to permanently disable index " + index.getName().getString(), e);
+ }
+ }
+ }
+ } catch (Exception e) {
+ // If we can't reach the stats table, don't interrupt the normal
+ // compaction operation, just log a warning.
+ if (LOG.isWarnEnabled()) {
+ LOG.warn("Unable to permanently disable indexes being partially rebuild for " + fullTableName, e);
+ }
+ }
+ }
+ return internalScanner;
+ }
+ });
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b8595f1a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 75a7e55..ea6fb96 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -711,10 +711,15 @@ public class IndexUtil {
mutations.toArray(new Mutation[mutations.size()]),
HConstants.NO_NONCE, HConstants.NO_NONCE);
}
-
+
public static MetaDataMutationResult updateIndexState(String indexTableName, long minTimeStamp,
HTableInterface metaTable, PIndexState newState) throws Throwable {
byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName);
+ return updateIndexState(indexTableKey, minTimeStamp, metaTable, newState);
+ }
+
+ public static MetaDataMutationResult updateIndexState(byte[] indexTableKey, long minTimeStamp,
+ HTableInterface metaTable, PIndexState newState) throws Throwable {
// Mimic the Put that gets generated by the client on an update of the index state
Put put = new Put(indexTableKey);
put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES,