You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@phoenix.apache.org by GitBox <gi...@apache.org> on 2020/07/02 21:24:13 UTC
[GitHub] [phoenix] gjacoby126 opened a new pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
gjacoby126 opened a new pull request #821:
URL: https://github.com/apache/phoenix/pull/821
…(#809)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] gjacoby126 commented on a change in pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
gjacoby126 commented on a change in pull request #821:
URL: https://github.com/apache/phoenix/pull/821#discussion_r449312574
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
##########
@@ -791,95 +826,107 @@ public void testIndexToolForIncrementalVerify() throws Exception {
conn.createStatement().execute(String.format(
"CREATE INDEX "+indexTableName+" ON "+dataTableFullName+" (val3) INCLUDE(val5)"));
- customeEdge.setValue(EnvironmentEdgeManager.currentTimeMillis());
- EnvironmentEdgeManager.injectEdge(customeEdge);
- long t0 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val1, val2) VALUES (4,5,'abc',3)");
- customeEdge.incrementValue(waitForUpsert);
- long t1 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val1, val2) VALUES (1,2,'abc',3)");
- customeEdge.incrementValue(waitForUpsert);
- long t2 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val3, val4) VALUES (1,2,4,1.2)");
- customeEdge.incrementValue(waitForUpsert);
- long t3 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val5, val6) VALUES (1,2,5,'def')");
- customeEdge.incrementValue(waitForUpsert);
- long t4 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("DELETE FROM "+dataTableFullName+" WHERE key1=4");
- customeEdge.incrementValue(waitForUpsert);
- long t5 = customeEdge.currentTime();
- customeEdge.incrementValue(10);
- long t6 = customeEdge.currentTime();
+ customEdge.setValue(EnvironmentEdgeManager.currentTimeMillis());
+ EnvironmentEdgeManager.injectEdge(customEdge);
+ long t0 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val1, val2) VALUES (4,5,'abc',3)");
+ customEdge.incrementValue(waitForUpsert);
+ long t1 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val1, val2) VALUES (1,2,'abc',3)");
+ customEdge.incrementValue(waitForUpsert);
+ long t2 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val3, val4) VALUES (1,2,4,1.2)");
+ customEdge.incrementValue(waitForUpsert);
+ long t3 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val5, val6) VALUES (1,2,5,'def')");
+ customEdge.incrementValue(waitForUpsert);
+ long t4 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("DELETE FROM "+viewFullName+" WHERE key1=4");
+ customEdge.incrementValue(waitForUpsert);
+ long t5 = customEdge.currentTime();
+ customEdge.incrementValue(10);
+ long t6 = customEdge.currentTime();
IndexTool it;
if(!mutable) {
// job with 2 rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t2));
verifyCounters(it, 2, 2);
+ //increment time between rebuilds so that PHOENIX_INDEX_TOOL and
+ // PHOENIX_INDEX_TOOL_RESULT tables get unique keys for each run
+ customEdge.incrementValue(waitForUpsert);
// only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t2));
verifyCounters(it, 1, 1);
-
+ customEdge.incrementValue(waitForUpsert);
// no rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t5),"-et", String.valueOf(t6));
verifyCounters(it, 0, 0);
-
+ customEdge.incrementValue(waitForUpsert);
//view index
// job with 2 rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, viewIndexName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t2));
verifyCounters(it, 2, 2);
-
+ customEdge.incrementValue(waitForUpsert);
// only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, viewIndexName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t2));
verifyCounters(it, 1, 1);
-
+ customEdge.incrementValue(waitForUpsert);
// no rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, viewIndexName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t5),"-et", String.valueOf(t6));
verifyCounters(it, 0, 0);
-
+ customEdge.incrementValue(waitForUpsert);
return;
}
+
// regular job without delete row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t4));
- verifyCounters(it, 2, 3);
+ verifyCounters(it, 2, 2);
+ customEdge.incrementValue(waitForUpsert);
// job with 2 rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t2));
verifyCounters(it, 2, 2);
+ customEdge.incrementValue(waitForUpsert);
// job with update on only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t3));
- verifyCounters(it, 1, 2);
+ verifyCounters(it, 1, 1);
Review comment:
There should be 2 versions of a single row, I believe. Different max versions behavior coming from lack of max lookback protection maybe?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] swaroopak commented on a change in pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
swaroopak commented on a change in pull request #821:
URL: https://github.com/apache/phoenix/pull/821#discussion_r449301691
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
##########
@@ -791,95 +826,107 @@ public void testIndexToolForIncrementalVerify() throws Exception {
conn.createStatement().execute(String.format(
"CREATE INDEX "+indexTableName+" ON "+dataTableFullName+" (val3) INCLUDE(val5)"));
- customeEdge.setValue(EnvironmentEdgeManager.currentTimeMillis());
- EnvironmentEdgeManager.injectEdge(customeEdge);
- long t0 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val1, val2) VALUES (4,5,'abc',3)");
- customeEdge.incrementValue(waitForUpsert);
- long t1 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val1, val2) VALUES (1,2,'abc',3)");
- customeEdge.incrementValue(waitForUpsert);
- long t2 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val3, val4) VALUES (1,2,4,1.2)");
- customeEdge.incrementValue(waitForUpsert);
- long t3 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("UPSERT INTO "+dataTableFullName+"(key1, key2, val5, val6) VALUES (1,2,5,'def')");
- customeEdge.incrementValue(waitForUpsert);
- long t4 = customeEdge.currentTime();
- customeEdge.incrementValue(waitForUpsert);
- conn.createStatement().execute("DELETE FROM "+dataTableFullName+" WHERE key1=4");
- customeEdge.incrementValue(waitForUpsert);
- long t5 = customeEdge.currentTime();
- customeEdge.incrementValue(10);
- long t6 = customeEdge.currentTime();
+ customEdge.setValue(EnvironmentEdgeManager.currentTimeMillis());
+ EnvironmentEdgeManager.injectEdge(customEdge);
+ long t0 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val1, val2) VALUES (4,5,'abc',3)");
+ customEdge.incrementValue(waitForUpsert);
+ long t1 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val1, val2) VALUES (1,2,'abc',3)");
+ customEdge.incrementValue(waitForUpsert);
+ long t2 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val3, val4) VALUES (1,2,4,1.2)");
+ customEdge.incrementValue(waitForUpsert);
+ long t3 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("UPSERT INTO "+viewFullName+"(key1, key2, val5, val6) VALUES (1,2,5,'def')");
+ customEdge.incrementValue(waitForUpsert);
+ long t4 = customEdge.currentTime();
+ customEdge.incrementValue(waitForUpsert);
+ conn.createStatement().execute("DELETE FROM "+viewFullName+" WHERE key1=4");
+ customEdge.incrementValue(waitForUpsert);
+ long t5 = customEdge.currentTime();
+ customEdge.incrementValue(10);
+ long t6 = customEdge.currentTime();
IndexTool it;
if(!mutable) {
// job with 2 rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t2));
verifyCounters(it, 2, 2);
+ //increment time between rebuilds so that PHOENIX_INDEX_TOOL and
+ // PHOENIX_INDEX_TOOL_RESULT tables get unique keys for each run
+ customEdge.incrementValue(waitForUpsert);
// only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t2));
verifyCounters(it, 1, 1);
-
+ customEdge.incrementValue(waitForUpsert);
// no rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t5),"-et", String.valueOf(t6));
verifyCounters(it, 0, 0);
-
+ customEdge.incrementValue(waitForUpsert);
//view index
// job with 2 rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, viewIndexName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t2));
verifyCounters(it, 2, 2);
-
+ customEdge.incrementValue(waitForUpsert);
// only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, viewIndexName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t2));
verifyCounters(it, 1, 1);
-
+ customEdge.incrementValue(waitForUpsert);
// no rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, viewIndexName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t5),"-et", String.valueOf(t6));
verifyCounters(it, 0, 0);
-
+ customEdge.incrementValue(waitForUpsert);
return;
}
+
// regular job without delete row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t4));
- verifyCounters(it, 2, 3);
+ verifyCounters(it, 2, 2);
+ customEdge.incrementValue(waitForUpsert);
// job with 2 rows
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t2));
verifyCounters(it, 2, 2);
+ customEdge.incrementValue(waitForUpsert);
// job with update on only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t3));
- verifyCounters(it, 1, 2);
+ verifyCounters(it, 1, 1);
Review comment:
There are 2 index rows in this time window. Wondering if it is returning 1 because of different skip scan behaviors in the master branch's HBase or something else? Do you know @abhishek-chouhan?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] gjacoby126 commented on a change in pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
gjacoby126 commented on a change in pull request #821:
URL: https://github.com/apache/phoenix/pull/821#discussion_r450476871
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
##########
@@ -952,20 +998,22 @@ public void testIndexToolForIncrementalVerify_viewIndex() throws Exception {
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),
"-et", String.valueOf(t3));
verifyCounters(it, 2, 2);
-
+/*
// job with update on only one row
it =
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t3),
"-et", String.valueOf(t4));
verifyCounters(it, 1, 1);
Review comment:
Based on our previous conversation that led to the filing of PHOENIX-5989, I've just updated this with a reference to 5989 and kept it commented out.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] gjacoby126 merged pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
gjacoby126 merged pull request #821:
URL: https://github.com/apache/phoenix/pull/821
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] gjacoby126 commented on a change in pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
gjacoby126 commented on a change in pull request #821:
URL: https://github.com/apache/phoenix/pull/821#discussion_r449311095
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
##########
@@ -952,20 +998,22 @@ public void testIndexToolForIncrementalVerify_viewIndex() throws Exception {
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),
"-et", String.valueOf(t3));
verifyCounters(it, 2, 2);
-
+/*
// job with update on only one row
it =
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t3),
"-et", String.valueOf(t4));
verifyCounters(it, 1, 1);
Review comment:
That seems like incorrect behavior though, because that delete statement _does_ delete a row that's part of the view.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] gjacoby126 commented on pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
gjacoby126 commented on pull request #821:
URL: https://github.com/apache/phoenix/pull/821#issuecomment-655157762
thanks for the review, @swaroopak
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [phoenix] swaroopak commented on a change in pull request #821: PHOENIX-5973 - Stabilize and speed up IndexToolForNonTxGlobalIndexIT …
Posted by GitBox <gi...@apache.org>.
swaroopak commented on a change in pull request #821:
URL: https://github.com/apache/phoenix/pull/821#discussion_r449300916
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
##########
@@ -952,20 +998,22 @@ public void testIndexToolForIncrementalVerify_viewIndex() throws Exception {
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),
"-et", String.valueOf(t3));
verifyCounters(it, 2, 2);
-
+/*
// job with update on only one row
it =
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t3),
"-et", String.valueOf(t4));
verifyCounters(it, 1, 1);
Review comment:
This should return 0 scanned and 0 valid because the view has where clause on val6 which won't qualify for the delete marker that is there on HBase row. Thank you @abhishek-chouhan for clarifying this.
That means in order to consider a deleted row for incremental verification on a view with where clause, its upsert should also fall into the same window. Otherwise, any such mismatches will be caught in full verification.
FYI @kadirozde @priyankporwal
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org