You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2017/08/08 05:53:12 UTC
[6/6] phoenix git commit: PHOENIX-3525 Cap automatic index rebuilding
to inactive timestamp
PHOENIX-3525 Cap automatic index rebuilding to inactive timestamp
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ecf185a2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ecf185a2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ecf185a2
Branch: refs/heads/4.x-HBase-0.98
Commit: ecf185a2980e358fcc35fea389d9f321d001edfb
Parents: e976922
Author: James Taylor <ja...@apache.org>
Authored: Mon Aug 7 19:06:46 2017 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Mon Aug 7 22:50:03 2017 -0700
----------------------------------------------------------------------
.../end2end/index/MutableIndexFailureIT.java | 1 +
.../end2end/index/PartialIndexRebuilderIT.java | 1 +
.../coprocessor/MetaDataEndpointImpl.java | 18 +++
.../coprocessor/MetaDataRegionObserver.java | 102 +++++++++++------
.../index/covered/data/LazyValueGetter.java | 113 ++++++++++---------
.../org/apache/phoenix/query/QueryServices.java | 5 +
.../phoenix/query/QueryServicesOptions.java | 3 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 5 -
.../java/org/apache/phoenix/util/TestUtil.java | 3 +-
9 files changed, 155 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 34f95e2..bc5237a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -136,6 +136,7 @@ public class MutableIndexFailureIT extends BaseTest {
serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "30000"); // give up rebuilding after 30 seconds
// need to override rpc retries otherwise test doesn't pass
serverProps.put(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, Long.toString(1));
+ serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(1000));
Map<String, String> clientProps = Collections.singletonMap(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
NUM_SLAVES_BASE = 4;
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
index ef9ae1b..cc5f138 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
@@ -51,6 +51,7 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT {
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString());
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, "1000");
serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "30000"); // give up rebuilding after 30 seconds
+ serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(1000));
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), ReadOnlyProps.EMPTY_PROPS);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index a153c54..dfdc28d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -3516,6 +3516,24 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
Cell newDisableTimeStampCell = newKVs.get(disableTimeStampKVIndex);
long newDisableTimeStamp = (Long) PLong.INSTANCE.toObject(newDisableTimeStampCell.getValueArray(),
newDisableTimeStampCell.getValueOffset(), newDisableTimeStampCell.getValueLength());
+ // We never set the INDEX_DISABLE_TIMESTAMP to a positive value when we're setting the state to ACTIVE.
+ // Instead, we're passing in what we expect the INDEX_DISABLE_TIMESTAMP to be currently. If it's
+ // changed, it means that a data table row failed to write while we were partially rebuilding it
+ // and we must rerun it.
+ if (newState == PIndexState.ACTIVE && newDisableTimeStamp > 0) {
+ // Don't allow setting to ACTIVE if the INDEX_DISABLE_TIMESTAMP doesn't match
+ // what we expect.
+ if (newDisableTimeStamp != Math.abs(curTimeStampVal)) {
+ builder.setReturnCode(MetaDataProtos.MutationCode.UNALLOWED_TABLE_MUTATION);
+ builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis());
+ done.run(builder.build());
+ return;
+ }
+ // Reset INDEX_DISABLE_TIMESTAMP_BYTES to zero as we're good to go.
+ newKVs.set(disableTimeStampKVIndex,
+ CellUtil.createCell(key, TABLE_FAMILY_BYTES, INDEX_DISABLE_TIMESTAMP_BYTES,
+ timeStamp, KeyValue.Type.Put.getCode(), PLong.INSTANCE.toBytes(0L)));
+ }
// We use the sign of the INDEX_DISABLE_TIMESTAMP to differentiate the keep-index-active (negative)
// from block-writes-to-data-table case. In either case, we want to keep the oldest timestamp to
// drive the partial index rebuild rather than update it with each attempt to update the index
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
index ae6c198..063689f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -54,6 +55,7 @@ import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.phoenix.cache.GlobalCache;
@@ -206,6 +208,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
private long rebuildIndexBatchSize = HConstants.LATEST_TIMESTAMP;
private long configuredBatches = 10;
private long indexDisableTimestampThreshold;
+ private final ReadOnlyProps props;
public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) {
this.env = env;
@@ -217,6 +220,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
this.indexDisableTimestampThreshold =
configuration.getLong(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD,
QueryServicesOptions.DEFAULT_INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD);
+ this.props = new ReadOnlyProps(env.getConfiguration().iterator());
}
@Override
@@ -248,7 +252,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
- Map<PTable, List<PTable>> dataTableToIndexesMap = null;
+ Map<PTable, List<Pair<PTable,Long>>> dataTableToIndexesMap = null;
boolean hasMore = false;
List<Cell> results = new ArrayList<Cell>();
scanner = this.env.getRegion().getScanner(scan);
@@ -261,20 +265,23 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
Result r = Result.create(results);
byte[] disabledTimeStamp = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
- byte[] indexState = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
- PhoenixDatabaseMetaData.INDEX_STATE_BYTES);
+ Cell indexStateCell = r.getColumnLatestCell(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES);
if (disabledTimeStamp == null || disabledTimeStamp.length == 0) {
continue;
}
+ long indexDisableTimestamp =
+ PLong.INSTANCE.getCodec().decodeLong(disabledTimeStamp, 0,
+ SortOrder.ASC);
byte[] dataTable = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES);
- if ((dataTable == null || dataTable.length == 0) || (indexState == null || indexState.length == 0)) {
+ if ((dataTable == null || dataTable.length == 0) || indexStateCell == null) {
// data table name can't be empty
continue;
}
+ byte[] indexState = CellUtil.cloneValue(indexStateCell);
byte[][] rowKeyMetaData = new byte[3][];
SchemaUtil.getVarChars(r.getRow(), 3, rowKeyMetaData);
byte[] schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
@@ -306,12 +313,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
+ indexPTable.getName() + " are online.");
continue;
}
- long indexDisableTimestamp =
- PLong.INSTANCE.getCodec().decodeLong(disabledTimeStamp, 0,
- SortOrder.ASC);
- PIndexState state = PIndexState.fromSerializedValue(indexState[0]);
- if (indexDisableTimestamp > 0 && System.currentTimeMillis()
- - indexDisableTimestamp > indexDisableTimestampThreshold) {
+ if (System.currentTimeMillis() - Math.abs(indexDisableTimestamp) > indexDisableTimestampThreshold) {
/*
* It has been too long since the index has been disabled and any future
* attempts to reenable it likely will fail. So we are going to mark the
@@ -335,8 +337,28 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
// cannot transition directly from DISABLED -> ACTIVE
if (Bytes.compareTo(PIndexState.DISABLE.getSerializedBytes(), indexState) == 0) {
IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.INACTIVE, null);
+ continue; // Must wait until clients start to do index maintenance again
+ }
+ long currentTime = System.currentTimeMillis();
+ long forwardOverlapDurationMs = env.getConfiguration().getLong(
+ QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB,
+ QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME);
+ // Wait until no failures have occurred in at least forwardOverlapDurationMs
+ if (indexStateCell.getTimestamp() + forwardOverlapDurationMs > currentTime) {
+ continue; // Haven't waited long enough yet
+ }
+ Long upperBoundOfRebuild = HConstants.LATEST_TIMESTAMP;
+ if (Bytes.compareTo(PIndexState.INACTIVE.getSerializedBytes(), indexState) == 0) {
+ upperBoundOfRebuild = indexStateCell.getTimestamp() + forwardOverlapDurationMs;
+ } else if (Bytes.compareTo(PIndexState.ACTIVE.getSerializedBytes(), indexState) == 0) {
+ // Since the index state cell is updated every time the INDEX_DISABLED_TIMESTAMP
+ // changes, we know the upper bound.
+ upperBoundOfRebuild = indexStateCell.getTimestamp() + 1;
+ } else {
+ LOG.warn("Unexpected index state of " + indexTableFullName + ":" + Bytes.toStringBinary(indexState));
+ continue; // Ignore as this is an unexpected state
}
- List<PTable> indexesToPartiallyRebuild = dataTableToIndexesMap.get(dataPTable);
+ List<Pair<PTable,Long>> indexesToPartiallyRebuild = dataTableToIndexesMap.get(dataPTable);
if (indexesToPartiallyRebuild == null) {
indexesToPartiallyRebuild = Lists.newArrayListWithExpectedSize(dataPTable.getIndexes().size());
dataTableToIndexesMap.put(dataPTable, indexesToPartiallyRebuild);
@@ -344,24 +366,26 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
LOG.debug("We have found " + indexPTable.getIndexState() + " Index:" + indexPTable.getName()
+ " on data table:" + dataPTable.getName() + " which failed to be updated at "
+ indexPTable.getIndexDisableTimestamp());
- indexesToPartiallyRebuild.add(indexPTable);
+ indexesToPartiallyRebuild.add(new Pair<PTable,Long>(indexPTable,upperBoundOfRebuild));
} while (hasMore);
if (dataTableToIndexesMap != null) {
- long overlapTime = env.getConfiguration().getLong(
- QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB,
- QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME);
- for (Map.Entry<PTable, List<PTable>> entry : dataTableToIndexesMap.entrySet()) {
+ long backwardOverlapDurationMs = env.getConfiguration().getLong(
+ QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME_ATTRIB,
+ env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB,
+ QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME));
+ for (Map.Entry<PTable, List<Pair<PTable,Long>>> entry : dataTableToIndexesMap.entrySet()) {
PTable dataPTable = entry.getKey();
- List<PTable> indexesToPartiallyRebuild = entry.getValue();
- ReadOnlyProps props = new ReadOnlyProps(env.getConfiguration().iterator());
+ List<Pair<PTable,Long>> pairs = entry.getValue();
+ List<PTable> indexesToPartiallyRebuild = Lists.newArrayListWithExpectedSize(pairs.size());
try (HTableInterface metaTable = env.getTable(
SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props))) {
long earliestDisableTimestamp = Long.MAX_VALUE;
+ long latestUpperBoundTimestamp = Long.MIN_VALUE;
List<IndexMaintainer> maintainers = Lists
- .newArrayListWithExpectedSize(indexesToPartiallyRebuild.size());
+ .newArrayListWithExpectedSize(pairs.size());
int signOfDisableTimeStamp = 0;
- for (PTable index : indexesToPartiallyRebuild) {
+ for (Pair<PTable,Long> pair : pairs) {
// We need a way of differentiating the block writes to data table case from
// the leave index active case. In either case, we need to know the time stamp
// at which writes started failing so we can rebuild from that point. If we
@@ -369,6 +393,8 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
// then writes to the data table will be blocked (this is client side logic
// and we can't change this in a minor release). So we use the sign of the
// time stamp to differentiate.
+ PTable index = pair.getFirst();
+ Long upperBoundTimestamp = pair.getSecond();
long disabledTimeStampVal = index.getIndexDisableTimestamp();
if (disabledTimeStampVal != 0) {
if (signOfDisableTimeStamp != 0 && signOfDisableTimeStamp != Long.signum(disabledTimeStampVal)) {
@@ -380,16 +406,20 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
earliestDisableTimestamp = disabledTimeStampVal;
}
+ indexesToPartiallyRebuild.add(index);
maintainers.add(index.getIndexMaintainer(dataPTable, conn));
}
+ if (upperBoundTimestamp > latestUpperBoundTimestamp) {
+ latestUpperBoundTimestamp = upperBoundTimestamp;
+ }
}
// No indexes are disabled, so skip this table
if (earliestDisableTimestamp == Long.MAX_VALUE) {
continue;
}
- long timeStamp = Math.max(0, earliestDisableTimestamp - overlapTime);
+ long scanBeginTime = Math.max(0, earliestDisableTimestamp - backwardOverlapDurationMs);
LOG.info("Starting to build " + dataPTable + " indexes " + indexesToPartiallyRebuild
- + " from timestamp=" + timeStamp);
+ + " from timestamp=" + scanBeginTime + " until " + latestUpperBoundTimestamp);
TableRef tableRef = new TableRef(null, dataPTable, HConstants.LATEST_TIMESTAMP, false);
// TODO Need to set high timeout
@@ -399,10 +429,10 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
Scan dataTableScan = IndexManagementUtil.newLocalStateScan(plan.getContext().getScan(),
maintainers);
- long scanEndTime = getTimestampForBatch(timeStamp,
- batchExecutedPerTableMap.get(dataPTable.getName()));
+ long scanEndTime = Math.min(latestUpperBoundTimestamp,
+ getTimestampForBatch(scanBeginTime,batchExecutedPerTableMap.get(dataPTable.getName())));
// We can't allow partial results
- dataTableScan.setTimeRange(timeStamp, scanEndTime);
+ dataTableScan.setTimeRange(scanBeginTime, scanEndTime);
dataTableScan.setCacheBlocks(false);
dataTableScan.setAttribute(BaseScannerRegionObserver.REBUILD_INDEXES, TRUE_BYTES);
@@ -418,7 +448,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
+ (scanEndTime == HConstants.LATEST_TIMESTAMP ? "LATEST_TIMESTAMP" : scanEndTime));
MutationState mutationState = plan.execute();
long rowCount = mutationState.getUpdateCount();
- if (scanEndTime == HConstants.LATEST_TIMESTAMP) {
+ if (scanEndTime == latestUpperBoundTimestamp) {
LOG.info("Rebuild completed for all inactive/disabled indexes in data table:"
+ dataPTable.getName());
}
@@ -427,13 +457,17 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
String indexTableFullName = SchemaUtil.getTableName(
indexPTable.getSchemaName().getString(),
indexPTable.getTableName().getString());
- if (scanEndTime == HConstants.LATEST_TIMESTAMP) {
- IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.ACTIVE, 0l);
+ if (scanEndTime == latestUpperBoundTimestamp) {
+ // We compare the absolute value of the index disable timestamp. We don't want to
+ // pass a negative value because that means an additional index write failed.
+ IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.ACTIVE, Math.abs(indexPTable.getIndexDisableTimestamp()));
batchExecutedPerTableMap.remove(dataPTable.getName());
LOG.info("Making Index:" + indexPTable.getTableName() + " active after rebuilding");
} else {
// Maintain sign of INDEX_DISABLE_TIMESTAMP (see comment above)
- updateDisableTimestamp(conn, indexTableFullName, env, scanEndTime * signOfDisableTimeStamp, metaTable);
+ if (!updateDisableTimestamp(conn, indexTableFullName, scanEndTime * signOfDisableTimeStamp, metaTable, indexPTable.getIndexDisableTimestamp())) {
+ LOG.warn("The index disabled timestamp for " + indexTableFullName + " was updated outside of rebuilder. Will reattempt rebuild next iteration.");
+ }
Long noOfBatches = batchExecutedPerTableMap.get(dataPTable.getName());
if (noOfBatches == null) {
noOfBatches = 0l;
@@ -445,8 +479,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
dataPTable.getTableName().getString());
new MetaDataClient(conn).updateCache(indexPTable.getSchemaName().getString(),
indexPTable.getTableName().getString());
- LOG.info(
- "During Round-robin build: Successfully updated index disabled timestamp for "
+ LOG.info("During Round-robin build: Successfully updated index disabled timestamp for "
+ indexTableFullName + " to " + scanEndTime);
}
}
@@ -490,8 +523,8 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
}
}
- private static void updateDisableTimestamp(PhoenixConnection conn, String indexTableName,
- RegionCoprocessorEnvironment env, long disabledTimestamp, HTableInterface metaTable) throws IOException {
+ private static boolean updateDisableTimestamp(PhoenixConnection conn, String indexTableName,
+ long disabledTimestamp, HTableInterface metaTable, long expectedDisabledTimestamp) throws IOException {
byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName);
Put put = new Put(indexTableKey);
put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES,
@@ -501,6 +534,9 @@ public class MetaDataRegionObserver extends BaseRegionObserver {
metaTable.checkAndMutate(indexTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, CompareOp.NOT_EQUAL, PLong.INSTANCE.toBytes(0),
rowMutations);
+ return metaTable.checkAndPut(indexTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, PLong.INSTANCE.toBytes(expectedDisabledTimestamp),
+ put);
}
private static synchronized void initRebuildIndexConnectionProps(Configuration config) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
index d163acf..05ebeca 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/data/LazyValueGetter.java
@@ -39,67 +39,70 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
*/
public class LazyValueGetter implements ValueGetter {
- private CoveredDeleteScanner scan;
- private volatile Map<ColumnReference, ImmutableBytesWritable> values;
- private byte[] row;
-
- /**
- * Back the getter with a {@link Scanner} to actually access the local data.
- * @param scan backing scanner
- * @param currentRow row key for the row to seek in the scanner
- */
- public LazyValueGetter(CoveredDeleteScanner scan, byte[] currentRow) {
- this.scan = scan;
- this.row = currentRow;
- }
+ private CoveredDeleteScanner scan;
+ private volatile Map<ColumnReference, ImmutableBytesWritable> values;
+ private byte[] row;
- @Override
- public ImmutableBytesWritable getLatestValue(ColumnReference ref, long ts) throws IOException {
- // ensure we have a backing map
- if (values == null) {
- synchronized (this) {
- values = Collections.synchronizedMap(new HashMap<ColumnReference, ImmutableBytesWritable>());
- }
+ /**
+ * Back the getter with a {@link Scanner} to actually access the local data.
+ * @param scan backing scanner
+ * @param currentRow row key for the row to seek in the scanner
+ */
+ public LazyValueGetter(CoveredDeleteScanner scan, byte[] currentRow) {
+ this.scan = scan;
+ this.row = currentRow;
}
- // check the value in the map
- ImmutableBytesWritable value = values.get(ref);
- if (value == null) {
- value = get(ref);
- DeleteTracker deleteTracker = scan.getDeleteTracker();
- if (value == null) {
- // Delete family is used for row deletion. Family won't necessarily match as we'll be at
- // the delete family marker on the last column family if there is one.
- if (deleteTracker.deleteFamily != null && deleteTracker.deleteFamily.getTimestamp() == ts) {
- value = HIDDEN_BY_DELETE;
- }
- }
- values.put(ref, value);
- }
+ @Override
+ public ImmutableBytesWritable getLatestValue(ColumnReference ref, long ts) throws IOException {
+ Map<ColumnReference, ImmutableBytesWritable> v = values;
+ // ensure we have a backing map
+ if (v == null) {
+ synchronized (this) {
+ v = values;
+ if (v == null) {
+ v = values = Collections.synchronizedMap(new HashMap<ColumnReference, ImmutableBytesWritable>());
+ }
+ }
+ }
- return value;
- }
+ // check the value in the map
+ ImmutableBytesWritable value = v.get(ref);
+ if (value == null) {
+ value = get(ref);
+ DeleteTracker deleteTracker = scan.getDeleteTracker();
+ if (value == null) {
+ // Delete family is used for row deletion. Family won't necessarily match as we'll be at
+ // the delete family marker on the last column family if there is one.
+ if (deleteTracker.deleteFamily != null && deleteTracker.deleteFamily.getTimestamp() == ts) {
+ value = HIDDEN_BY_DELETE;
+ }
+ }
+ v.put(ref, value);
+ }
- /**
- * @param ref
- * @return the first value on the scanner for the given column
- */
- @SuppressWarnings("deprecation")
- private ImmutableBytesPtr get(ColumnReference ref) throws IOException {
- KeyValue first = ref.getFirstKeyValueForRow(row);
- if (!scan.seek(first)) {
- return null;
+ return value;
}
- // there is a next value - we only care about the current value, so we can just snag that
- Cell next = scan.next();
- if (ref.matches(KeyValueUtil.ensureKeyValue(next))) {
- return new ImmutableBytesPtr(next.getValueArray(), next.getValueOffset(), next.getValueLength());
+
+ /**
+ * @param ref
+ * @return the first value on the scanner for the given column
+ */
+ private ImmutableBytesPtr get(ColumnReference ref) throws IOException {
+ KeyValue first = ref.getFirstKeyValueForRow(row);
+ if (!scan.seek(first)) {
+ return null;
+ }
+ // there is a next value - we only care about the current value, so we can just snag that
+ Cell next = scan.next();
+ if (ref.matches(KeyValueUtil.ensureKeyValue(next))) {
+ return new ImmutableBytesPtr(next.getValueArray(), next.getValueOffset(), next.getValueLength());
+ }
+ return null;
}
- return null;
- }
- @Override
- public byte[] getRowKey() {
- return this.row;
- }
+ @Override
+ public byte[] getRowKey() {
+ return this.row;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 7df90f4..1052cbe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -150,8 +150,13 @@ public interface QueryServices extends SQLCloseable {
public static final String INDEX_FAILURE_THROW_EXCEPTION_ATTRIB = "phoenix.index.failure.throw.exception";
// Index will be partially re-built from index disable time stamp - following overlap time
+ @Deprecated
public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB =
"phoenix.index.failure.handling.rebuild.overlap.time";
+ public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME_ATTRIB =
+ "phoenix.index.failure.handling.rebuild.overlap.backward.time";
+ public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB =
+ "phoenix.index.failure.handling.rebuild.overlap.forward.time";
public static final String INDEX_PRIOIRTY_ATTRIB = "phoenix.index.rpc.priority";
public static final String METADATA_PRIOIRTY_ATTRIB = "phoenix.metadata.rpc.priority";
public static final String ALLOW_LOCAL_INDEX_ATTRIB = "phoenix.index.allowLocalIndex";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 55cbc91..39dab7c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -172,7 +172,8 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_INDEX_FAILURE_DISABLE_INDEX = true;
public static final boolean DEFAULT_INDEX_FAILURE_THROW_EXCEPTION = true;
public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL = 60000; // 60 secs
- public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME = 1; // 1 ms
+ public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME = 1; // 1 ms
+ public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME = 60000 * 3; // 3 mins
// 30 min rpc timeout * 5 tries, with 2100ms total pause time between retries
public static final long DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT = (5 * 30000 * 60) + 2100;
public static final long DEFAULT_INDEX_REBUILD_RPC_TIMEOUT = 30000 * 60; // 30 mins
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index cc84b9a..9e5fc57 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -113,7 +113,6 @@ import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.transaction.PhoenixTransactionContext;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
public class IndexUtil {
@@ -791,10 +790,6 @@ public class IndexUtil {
public static void updateIndexState(PhoenixConnection conn, String indexTableName,
PIndexState newState, Long indexDisableTimestamp) throws SQLException {
- if (newState == PIndexState.ACTIVE) {
- Preconditions.checkArgument(indexDisableTimestamp == 0,
- "Index disable timestamp has to be 0 when marking an index as active");
- }
byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName);
String schemaName = SchemaUtil.getSchemaNameFromFullName(indexTableName);
String indexName = SchemaUtil.getTableNameFromFullName(indexTableName);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ecf185a2/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 266f4da..1716928 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -870,8 +870,7 @@ public class TestUtil {
+ "AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NULL AND " + PhoenixDatabaseMetaData.COLUMN_NAME + " IS NULL"
+ " AND " + PhoenixDatabaseMetaData.INDEX_STATE + " = '" + expectedIndexState.getSerializedValue() + "'";
ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- if (expectedIndexState == PIndexState.ACTIVE) {
+ if (rs.next() && expectedIndexState == PIndexState.ACTIVE) {
if (rs.getLong(1) == 0 && !rs.wasNull()) {
isActive = true;
break;