You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2019/10/01 20:32:21 UTC
[phoenix] branch master updated: PHOENIX-5503 IndexTool does not
rebuild all the rows
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new db2ae56 PHOENIX-5503 IndexTool does not rebuild all the rows
db2ae56 is described below
commit db2ae56ac02addb203172226bb4964eeeb1239ed
Author: Kadir <ko...@salesforce.com>
AuthorDate: Mon Sep 30 17:20:32 2019 -0700
PHOENIX-5503 IndexTool does not rebuild all the rows
---
.../java/org/apache/phoenix/end2end/IndexToolIT.java | 10 ++++++++++
.../org/apache/phoenix/mapreduce/index/IndexTool.java | 18 +++++++++---------
2 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index a284430..a4b7bdc 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -289,6 +289,16 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
assertEquals(NROWS, actualRowCount);
+
+ // Add more rows and make sure that these rows will be visible to IndexTool
+ for (int i = NROWS; i < 2 * NROWS; i++) {
+ upsertRow(stmt1, i);
+ }
+ conn.commit();
+ indexTool = runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, new String[0]);
+ assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
+ assertEquals(2 * NROWS, actualRowCount);
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index 229e4a1..7993602 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -297,10 +297,9 @@ public class IndexTool extends Configured implements Tool {
public Job getJob() throws Exception {
if (isPartialBuild) {
return configureJobForPartialBuild();
- } else {
+ } else if (useSnapshot || !useDirectApi || (!isLocalIndexBuild && pDataTable.isTransactional())) {
long maxTimeRange = pIndexTable.getTimeStamp() + 1;
// this is set to ensure index tables remains consistent post population.
-
if (pDataTable.isTransactional()) {
configuration.set(PhoenixConfigurationUtil.TX_SCN_VALUE,
Long.toString(TransactionUtil.convertToNanoseconds(maxTimeRange)));
@@ -308,13 +307,14 @@ public class IndexTool extends Configured implements Tool {
}
configuration.set(PhoenixConfigurationUtil.CURRENT_SCN_VALUE,
Long.toString(maxTimeRange));
- if (useSnapshot || !useDirectApi || (!isLocalIndexBuild && pDataTable.isTransactional())) {
- return configureJobForAysncIndex();
- }
- else {
- //Local and non-transactional global indexes to be built on the server side
- return configureJobForServerBuildIndex();
- }
+ return configureJobForAysncIndex();
+ }
+ else {
+ // Local and non-transactional global indexes to be built on the server side
+ // It is safe not to set CURRENT_SCN_VALUE for server side rebuilds, in order to make sure that
+ // all the rows that exist so far will be rebuilt. The current time of the servers will
+ // be used to set the time range for server side scans.
+ return configureJobForServerBuildIndex();
}
}