You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2017/07/17 18:24:46 UTC
phoenix git commit: PHOENIX-4028 Provide option to not throw index
write failure back to client
Repository: phoenix
Updated Branches:
refs/heads/master 7a83b8a1c -> 7220592ff
PHOENIX-4028 Provide option to not throw index write failure back to client
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7220592f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7220592f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7220592f
Branch: refs/heads/master
Commit: 7220592ff43235431c6d2909094e5e832665a73d
Parents: 7a83b8a
Author: James Taylor <ja...@apache.org>
Authored: Fri Jul 14 21:28:04 2017 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Mon Jul 17 11:24:15 2017 -0700
----------------------------------------------------------------------
.../apache/phoenix/end2end/AlterTableIT.java | 7 ++
.../apache/phoenix/end2end/CreateTableIT.java | 34 +++++-
.../end2end/index/MutableIndexFailureIT.java | 117 +++++++++----------
.../index/PhoenixIndexFailurePolicy.java | 17 ++-
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 1 +
6 files changed, 116 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7220592f/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
index 989472a..2cad013 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
@@ -1423,6 +1423,13 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
} catch (SQLException e) {
assertEquals(SQLExceptionCode.VIEW_WITH_PROPERTIES.getErrorCode(), e.getErrorCode());
}
+ ddl = "ALTER VIEW " + viewFullName + " SET THROW_INDEX_WRITE_FAILURE = FALSE";
+ try {
+ conn1.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.VIEW_WITH_PROPERTIES.getErrorCode(), e.getErrorCode());
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7220592f/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 5cc16a6..a3180a6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -49,7 +49,11 @@ import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.SchemaNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
-import org.apache.phoenix.util.*;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Assert;
import org.junit.Test;
@@ -760,4 +764,32 @@ public class CreateTableIT extends BaseClientManagedTimeIT {
conn.createStatement().execute("DROP SCHEMA " + NS);
}
}
+ @Test
+ public void testSetHTableDescriptorPropertyOnView() throws Exception {
+ long ts = nextTimestamp();
+ Properties props = new Properties();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts));
+ final String dataTableFullName = generateUniqueName();
+ String ddl = "CREATE TABLE " + dataTableFullName + " (\n"
+ +"ID1 VARCHAR(15) NOT NULL,\n"
+ +"ID2 VARCHAR(15) NOT NULL,\n"
+ +"CREATED_DATE DATE,\n"
+ +"CREATION_TIME BIGINT,\n"
+ +"LAST_USED DATE,\n"
+ +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) ";
+ Connection conn1 = DriverManager.getConnection(getUrl(), props);
+ conn1.createStatement().execute(ddl);
+ conn1.close();
+ final String viewFullName = generateUniqueName();
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+10));
+ Connection conn2 = DriverManager.getConnection(getUrl(), props);
+ ddl = "CREATE VIEW " + viewFullName + " AS SELECT * FROM " + dataTableFullName + " WHERE CREATION_TIME = 1 THROW_INDEX_WRITE_FAILURE = FALSE";
+ try {
+ conn2.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.VIEW_WITH_PROPERTIES.getErrorCode(), e.getErrorCode());
+ }
+ conn2.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7220592f/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index e3cac67..bc1a8b0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -99,15 +99,18 @@ public class MutableIndexFailureIT extends BaseTest {
private final boolean isNamespaceMapped;
private final boolean leaveIndexActiveOnFailure;
private final boolean rebuildIndexOnWriteFailure;
+ private final boolean failRebuildTask;
+ private final boolean throwIndexWriteFailure;
private String schema = generateUniqueName();
private List<CommitException> exceptions = Lists.newArrayList();
- public MutableIndexFailureIT(boolean transactional, boolean localIndex, boolean isNamespaceMapped, Boolean disableIndexOnWriteFailure, Boolean rebuildIndexOnWriteFailure) {
+ public MutableIndexFailureIT(boolean transactional, boolean localIndex, boolean isNamespaceMapped, Boolean disableIndexOnWriteFailure, Boolean rebuildIndexOnWriteFailure, boolean failRebuildTask, Boolean throwIndexWriteFailure) {
this.transactional = transactional;
this.localIndex = localIndex;
this.tableDDLOptions = " SALT_BUCKETS=2 " + (transactional ? ", TRANSACTIONAL=true " : "")
+ (disableIndexOnWriteFailure == null ? "" : (", " + PhoenixIndexFailurePolicy.DISABLE_INDEX_ON_WRITE_FAILURE + "=" + disableIndexOnWriteFailure))
- + (rebuildIndexOnWriteFailure == null ? "" : (", " + PhoenixIndexFailurePolicy.REBUILD_INDEX_ON_WRITE_FAILURE + "=" + rebuildIndexOnWriteFailure));
+ + (rebuildIndexOnWriteFailure == null ? "" : (", " + PhoenixIndexFailurePolicy.REBUILD_INDEX_ON_WRITE_FAILURE + "=" + rebuildIndexOnWriteFailure))
+ + (throwIndexWriteFailure == null ? "" : (", " + PhoenixIndexFailurePolicy.THROW_INDEX_WRITE_FAILURE + "=" + throwIndexWriteFailure));
this.tableName = FailingRegionObserver.FAIL_TABLE_NAME;
this.indexName = "A_" + FailingRegionObserver.FAIL_INDEX_NAME;
fullTableName = SchemaUtil.getTableName(schema, tableName);
@@ -115,6 +118,8 @@ public class MutableIndexFailureIT extends BaseTest {
this.isNamespaceMapped = isNamespaceMapped;
this.leaveIndexActiveOnFailure = ! (disableIndexOnWriteFailure == null ? QueryServicesOptions.DEFAULT_INDEX_FAILURE_DISABLE_INDEX : disableIndexOnWriteFailure);
this.rebuildIndexOnWriteFailure = ! Boolean.FALSE.equals(rebuildIndexOnWriteFailure);
+ this.failRebuildTask = failRebuildTask;
+ this.throwIndexWriteFailure = ! Boolean.FALSE.equals(throwIndexWriteFailure);
}
@BeforeClass
@@ -136,40 +141,37 @@ public class MutableIndexFailureIT extends BaseTest {
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
}
- @Parameters(name = "MutableIndexFailureIT_transactional={0},localIndex={1},isNamespaceMapped={2},disableIndexOnWriteFailure={3},rebuildIndexOnWriteFailure={4}") // name is used by failsafe as file name in reports
+ @Parameters(name = "MutableIndexFailureIT_transactional={0},localIndex={1},isNamespaceMapped={2},disableIndexOnWriteFailure={3},rebuildIndexOnWriteFailure={4},failRebuildTask={5},throwIndexWriteFailure={6}") // name is used by failsafe as file name in reports
public static List<Object[]> data() {
return Arrays.asList(new Object[][] {
- { false, false, true, true, true},
- { false, false, false, true, true},
- { true, false, false, true, true},
- { true, false, true, true, true},
- { false, true, true, true, true},
- { false, true, false, null, null},
- { true, true, false, true, null},
- { true, true, true, null, true},
-
- { false, false, false, false, true},
- { false, true, false, false, null},
- { false, false, false, false, false},
- { false, false, false, true, true},
- { false, false, false, true, true},
- { false, true, false, true, true},
- { false, true, false, true, true},
- }
+ { false, false, false, true, true, false, false},
+ { false, false, true, true, true, false, null},
+ { false, false, true, true, true, false, true},
+ { false, false, false, true, true, false, null},
+ { true, false, false, true, true, false, null},
+ { true, false, true, true, true, false, null},
+ { false, true, true, true, true, false, null},
+ { false, true, false, null, null, false, null},
+ { true, true, false, true, null, false, null},
+ { true, true, true, null, true, false, null},
+
+ { false, false, false, false, true, false, null},
+ { false, true, false, false, null, false, null},
+ { false, false, false, false, false, false, null},
+ { false, false, false, true, true, false, null},
+ { false, false, false, true, true, false, null},
+ { false, true, false, true, true, false, null},
+ { false, true, false, true, true, false, null},
+ { false, false, false, true, true, true, null},
+ { false, false, true, true, true, true, null},
+ { false, false, false, true, true, true, false},
+ { false, false, true, true, true, true, false},
+ }
);
}
@Test
- public void testWriteFailureDisablesIndex() throws Exception {
- helpTestWriteFailureDisablesIndex(false);
- }
-
- @Test
- public void testRebuildTaskFailureMarksIndexDisabled() throws Exception {
- helpTestWriteFailureDisablesIndex(true);
- }
-
- public void helpTestWriteFailureDisablesIndex(boolean failRebuildTask) throws Exception {
+ public void testIndexWriteFailure() throws Exception {
String secondIndexName = "B_" + FailingRegionObserver.FAIL_INDEX_NAME;
// String thirdIndexName = "C_" + INDEX_NAME;
// String thirdFullIndexName = SchemaUtil.getTableName(schema, thirdIndexName);
@@ -289,7 +291,7 @@ public class MutableIndexFailureIT extends BaseTest {
replayMutations();
}
- // Verify UPSERT on data table still works after index table is recreated
+ // Verify UPSERT on data table still works after index table is caught up
PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?,?)");
stmt.setString(1, "a3");
stmt.setString(2, "x4");
@@ -300,31 +302,28 @@ public class MutableIndexFailureIT extends BaseTest {
// verify index table has correct data (note that second index has been dropped)
validateDataWithIndex(conn, fullTableName, fullIndexName, localIndex);
} else {
- // the index is only disabled for non-txn tables upon index table write failure
- if (rebuildIndexOnWriteFailure && !transactional && !leaveIndexActiveOnFailure && !localIndex) {
- try {
- // Wait for index to be rebuilt automatically. This should fail because
- // we haven't flipped the FAIL_WRITE flag to false and as a result this
- // should cause index rebuild to fail too.
- waitForIndexToBeRebuilt(conn, indexName);
- // verify that the index was marked as disabled and the index disable
- // timestamp set to 0
- String q =
- "SELECT INDEX_STATE, INDEX_DISABLE_TIMESTAMP FROM SYSTEM.CATALOG WHERE TABLE_SCHEM = '"
- + schema + "' AND TABLE_NAME = '" + indexName + "'"
- + " AND COLUMN_NAME IS NULL AND COLUMN_FAMILY IS NULL";
- try (ResultSet r = conn.createStatement().executeQuery(q)) {
- assertTrue(r.next());
- assertEquals(PIndexState.DISABLE.getSerializedValue(), r.getString(1));
- assertEquals(0, r.getLong(2));
- assertFalse(r.next());
- }
- } finally {
- // even if the above test fails, make sure we leave the index active
- // as other tests might be dependent on it
- FAIL_WRITE = false;
- waitForIndexToBeRebuilt(conn, indexName);
+ try {
+ // Wait for index to be rebuilt automatically. This should fail because
+ // we haven't flipped the FAIL_WRITE flag to false and as a result this
+ // should cause index rebuild to fail too.
+ waitForIndexToBeRebuilt(conn, indexName);
+ // verify that the index was marked as disabled and the index disable
+ // timestamp set to 0
+ String q =
+ "SELECT INDEX_STATE, INDEX_DISABLE_TIMESTAMP FROM SYSTEM.CATALOG WHERE TABLE_SCHEM = '"
+ + schema + "' AND TABLE_NAME = '" + indexName + "'"
+ + " AND COLUMN_NAME IS NULL AND COLUMN_FAMILY IS NULL";
+ try (ResultSet r = conn.createStatement().executeQuery(q)) {
+ assertTrue(r.next());
+ assertEquals(PIndexState.DISABLE.getSerializedValue(), r.getString(1));
+ assertEquals(0, r.getLong(2));
+ assertFalse(r.next());
}
+ } finally {
+ // even if the above test fails, make sure we leave the index active
+ // as other tests might be dependent on it
+ FAIL_WRITE = false;
+ waitForIndexToBeRebuilt(conn, indexName);
}
}
} finally {
@@ -449,11 +448,11 @@ public class MutableIndexFailureIT extends BaseTest {
stmt.execute();
try {
conn.commit();
- if (commitShouldFail && !localIndex) {
+ if (commitShouldFail && !localIndex && this.throwIndexWriteFailure) {
fail();
}
} catch (CommitException e) {
- if (!commitShouldFail) {
+ if (!commitShouldFail || !this.throwIndexWriteFailure) {
throw e;
}
exceptions.add(e);
@@ -470,11 +469,11 @@ public class MutableIndexFailureIT extends BaseTest {
stmt.execute();
try {
conn.commit();
- if (commitShouldFail && !localIndex) {
+ if (commitShouldFail && !localIndex && this.throwIndexWriteFailure) {
fail();
}
} catch (CommitException e) {
- if (!commitShouldFail) {
+ if (!commitShouldFail || !this.throwIndexWriteFailure) {
throw e;
}
exceptions.add(e);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7220592f/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
index 842e881..a4f1f29 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
@@ -69,6 +69,7 @@ import com.google.common.collect.Multimap;
*/
public class PhoenixIndexFailurePolicy extends DelegateIndexFailurePolicy {
private static final Log LOG = LogFactory.getLog(PhoenixIndexFailurePolicy.class);
+ public static final String THROW_INDEX_WRITE_FAILURE = "THROW_INDEX_WRITE_FAILURE";
public static final String DISABLE_INDEX_ON_WRITE_FAILURE = "DISABLE_INDEX_ON_WRITE_FAILURE";
public static final String REBUILD_INDEX_ON_WRITE_FAILURE = "REBUILD_INDEX_ON_WRITE_FAILURE";
public static final String BLOCK_DATA_TABLE_WRITES_ON_WRITE_FAILURE = "BLOCK_DATA_TABLE_WRITES_ON_WRITE_FAILURE";
@@ -76,6 +77,7 @@ public class PhoenixIndexFailurePolicy extends DelegateIndexFailurePolicy {
private boolean blockDataTableWritesOnFailure;
private boolean disableIndexOnFailure;
private boolean rebuildIndexOnFailure;
+ private boolean throwIndexWriteFailure;
public PhoenixIndexFailurePolicy() {
super(new KillServerOnFailurePolicy());
@@ -110,6 +112,14 @@ public class PhoenixIndexFailurePolicy extends DelegateIndexFailurePolicy {
} else {
blockDataTableWritesOnFailure = Boolean.parseBoolean(value);
}
+
+ value = htd.getValue(THROW_INDEX_WRITE_FAILURE);
+ if (value == null) {
+ throwIndexWriteFailure = env.getConfiguration().getBoolean(QueryServices.INDEX_FAILURE_THROW_EXCEPTION_ATTRIB,
+ QueryServicesOptions.DEFAULT_INDEX_FAILURE_THROW_EXCEPTION);
+ } else {
+ throwIndexWriteFailure = Boolean.parseBoolean(value);
+ }
}
/**
@@ -135,7 +145,12 @@ public class PhoenixIndexFailurePolicy extends DelegateIndexFailurePolicy {
throwing = false;
} finally {
if (!throwing) {
- throw ServerUtil.wrapInDoNotRetryIOException("Unable to update the following indexes: " + attempted.keySet(), cause, timestamp);
+ IOException ioException = ServerUtil.wrapInDoNotRetryIOException("Unable to update the following indexes: " + attempted.keySet(), cause, timestamp);
+ if (throwIndexWriteFailure) {
+ throw ioException;
+ } else {
+ LOG.warn("Swallowing index write failure", ioException);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7220592f/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 57aba16..e13a527 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -148,6 +148,7 @@ public interface QueryServices extends SQLCloseable {
// Block writes to data table when index write fails
public static final String INDEX_FAILURE_BLOCK_WRITE = "phoenix.index.failure.block.write";
public static final String INDEX_FAILURE_DISABLE_INDEX = "phoenix.index.failure.disable.index";
+ public static final String INDEX_FAILURE_THROW_EXCEPTION_ATTRIB = "phoenix.index.failure.throw.exception";
// Index will be partially re-built from index disable time stamp - following overlap time
public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB =
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7220592f/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index b974d02..a59d5e6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -180,6 +180,7 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_INDEX_FAILURE_HANDLING_REBUILD = true; // auto rebuild on
public static final boolean DEFAULT_INDEX_FAILURE_BLOCK_WRITE = false;
public static final boolean DEFAULT_INDEX_FAILURE_DISABLE_INDEX = true;
+ public static final boolean DEFAULT_INDEX_FAILURE_THROW_EXCEPTION = true;
public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL = 60000; // 60 secs
public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME = 1; // 1 ms
public static final long DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT = 30000 * 60; // 30 mins