You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2016/02/23 23:55:09 UTC
[1/2] phoenix git commit: Revert "PHOENIX-2706 Implement client-side
mechanism to know if stats are enabled"
Repository: phoenix
Updated Branches:
refs/heads/master da51f46b2 -> c25f88791
Revert "PHOENIX-2706 Implement client-side mechanism to know if stats are enabled"
This reverts commit a1e6ae44be757b3fc5c4192ad55ecd8f3c03c01a.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dbc9ee9d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dbc9ee9d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dbc9ee9d
Branch: refs/heads/master
Commit: dbc9ee9dfe9e168c45ad279f8478c59f0882240c
Parents: da51f46
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Feb 23 09:36:53 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Feb 23 09:36:53 2016 -0800
----------------------------------------------------------------------
.../StatisticsCollectionRunTrackerIT.java | 10 ++--
.../org/apache/phoenix/end2end/IndexToolIT.java | 3 +-
.../phoenix/end2end/MutableIndexToolIT.java | 1 -
.../end2end/StatsCollectionDisabledIT.java | 19 ++-----
.../phoenix/end2end/StatsCollectorIT.java | 3 --
.../coprocessor/MetaDataEndpointImpl.java | 12 ++++-
.../UngroupedAggregateRegionObserver.java | 27 +++++-----
.../phoenix/iterate/BaseResultIterators.java | 25 +--------
.../phoenix/query/ConnectionQueryServices.java | 7 +--
.../query/ConnectionQueryServicesImpl.java | 25 ++-------
.../query/ConnectionlessQueryServicesImpl.java | 18 +------
.../query/DelegateConnectionQueryServices.java | 10 ----
.../org/apache/phoenix/query/QueryServices.java | 1 -
.../phoenix/query/QueryServicesOptions.java | 7 ---
.../apache/phoenix/schema/MetaDataClient.java | 2 -
.../schema/stats/NoOpStatisticsCollector.java | 3 ++
.../stats/StatisticsCollectionRunTracker.java | 7 +--
.../phoenix/schema/stats/StatisticsUtil.java | 16 ------
.../org/apache/phoenix/util/MetaDataUtil.java | 55 +++++++-------------
.../phoenix/query/QueryServicesTestImpl.java | 2 -
.../apache/phoenix/util/MetaDataUtilTest.java | 33 ------------
21 files changed, 71 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
index 345400e..bf567f0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
@@ -17,7 +17,8 @@
*/
package org.apache.phoenix.coprocessor;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_SKIPPED;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -37,6 +38,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -103,8 +105,7 @@ public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManaged
HRegionInfo regionInfo = createTableAndGetRegion(tableName);
// simulate stats collection via major compaction by marking the region as compacting in the tracker
markRegionAsCompacting(regionInfo);
- long returnValue = runUpdateStats(tableName);
- assertTrue("Update stats should have been skipped", returnValue >= UPDATE_STATS_SKIPPED);
+ Assert.assertEquals("Row count didn't match", COMPACTION_UPDATE_STATS_ROW_COUNT, runUpdateStats(tableName));
StatisticsCollectionRunTracker tracker =
StatisticsCollectionRunTracker.getInstance(new Configuration());
// assert that the tracker state was cleared.
@@ -116,7 +117,8 @@ public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManaged
String tableName = "testUpdateStatsPreventsAnotherUpdateStatsFromRunning".toUpperCase();
HRegionInfo regionInfo = createTableAndGetRegion(tableName);
markRunningUpdateStats(regionInfo);
- assertTrue("Update stats should have been skipped", runUpdateStats(tableName) >= UPDATE_STATS_SKIPPED);
+ Assert.assertEquals("Row count didn't match", CONCURRENT_UPDATE_STATS_ROW_COUNT,
+ runUpdateStats(tableName));
// assert that running the concurrent and race-losing update stats didn't clear the region
// from the tracker. If the method returned true it means the tracker was still tracking
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index fe95470..aba9c11 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -108,8 +108,7 @@ public class IndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String fullTableName = SchemaUtil.getTableName(schemaName, dataTable);
final String indxTable = String.format("%s_%s", dataTable, "INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
- props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
+ props.setProperty(QueryServices.TRANSACTIONS_ENABLED, "true");
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
index 8125007..0791479 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
@@ -61,7 +61,6 @@ public class MutableIndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String dataTable = "DATA_TABLE5";
final String indxTable = String.format("%s_%s",dataTable,"INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
index c3cdbc0..a92a665 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
@@ -17,27 +17,24 @@
*/
package org.apache.phoenix.end2end;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.Properties;
-import org.apache.phoenix.jdbc.PhoenixConnection;
+import com.google.common.collect.Maps;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.common.collect.Maps;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertFalse;
/**
* Verifies that statistics are not collected if they are disabled via a setting
@@ -57,21 +54,15 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
public void testStatisticsAreNotWritten() throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
- assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
Statement stmt = conn.createStatement();
- stmt.execute("CREATE TABLE T1 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR) SALT_BUCKETS=3");
+ stmt.execute("CREATE TABLE T1 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR)");
stmt.execute("UPSERT INTO T1 VALUES (1, 'NAME1')");
stmt.execute("UPSERT INTO T1 VALUES (2, 'NAME2')");
stmt.execute("UPSERT INTO T1 VALUES (3, 'NAME3')");
conn.commit();
stmt.execute("UPDATE STATISTICS T1");
- assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
ResultSet rs = stmt.executeQuery("SELECT * FROM SYSTEM.STATS");
assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("SELECT count(*) FROM T1");
- assertTrue(rs.next());
- assertEquals(3,rs.getInt(1));
- assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
rs.close();
stmt.close();
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index 6cd8e13..e72f41f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -94,7 +94,6 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
// props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
conn = DriverManager.getConnection(getUrl(), props);
- assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
conn.createStatement().execute(
"CREATE TABLE " + fullTableName +" ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC))"
@@ -105,7 +104,6 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// CAll the update statistics query here. If already major compaction has run this will not get executed.
stmt = conn.prepareStatement("UPDATE STATISTICS " + tableName);
stmt.execute();
- assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
stmt = upsertStmt(conn, tableName);
stmt.setString(1, "z");
s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
@@ -122,7 +120,6 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
stmt.execute();
rs = conn.createStatement().executeQuery("SELECT k FROM " + tableName);
assertTrue(rs.next());
- assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
conn.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 411ce03..ba7eb39 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -138,6 +138,7 @@ import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -2716,7 +2717,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
RpcCallback<GetVersionResponse> done) {
GetVersionResponse.Builder builder = GetVersionResponse.newBuilder();
- long version = MetaDataUtil.encodeVersion(env.getHBaseVersion(), env.getConfiguration());
+ // The first 3 bytes of the long is used to encoding the HBase version as major.minor.patch.
+ // The next 4 bytes of the value is used to encode the Phoenix version as major.minor.patch.
+ long version = MetaDataUtil.encodeHBaseAndPhoenixVersions(this.env.getHBaseVersion());
+
+ // The last byte is used to communicate whether or not mutable secondary indexing
+ // was configured properly.
+ version =
+ MetaDataUtil.encodeHasIndexWALCodec(version,
+ IndexManagementUtil.isWALEditCodecSet(this.env.getConfiguration()));
+
builder.setVersion(version);
done.run(builder.build());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 91ec20e..77b8b3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -22,9 +22,8 @@ import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_DISABLED;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_RUN;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_SKIPPED;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.exception.DataExceedsCapacityException;
@@ -90,7 +90,6 @@ import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.stats.StatisticsCollector;
import org.apache.phoenix.schema.stats.StatisticsCollectorFactory;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
import org.apache.phoenix.schema.types.PBinary;
import org.apache.phoenix.schema.types.PChar;
@@ -609,6 +608,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
InternalScanner internalScanner = scanner;
if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
try {
+ Pair<HRegionInfo, HRegionInfo> mergeRegions = null;
long clientTimeStamp = TimeKeeper.SYSTEM.getCurrentTime();
StatisticsCollector stats = StatisticsCollectorFactory.createStatisticsCollector(
c.getEnvironment(), table.getNameAsString(), clientTimeStamp,
@@ -643,23 +643,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
if (asyncBytes != null) {
async = Bytes.toBoolean(asyncBytes);
}
- long returnValue = UPDATE_STATS_RUN; // in case of async, we report 1 as number of rows updated
- boolean statsEnabled = StatisticsUtil.isStatsEnabled(config);
+ long rowCount = 0; // in case of async, we report 0 as number of rows updated
StatisticsCollectionRunTracker statsRunTracker =
StatisticsCollectionRunTracker.getInstance(config);
- boolean runUpdateStats = statsEnabled && statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
+ boolean runUpdateStats = statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
if (runUpdateStats) {
if (!async) {
- returnValue = callable.call();
+ rowCount = callable.call();
} else {
statsRunTracker.runTask(callable);
}
} else {
- returnValue = statsEnabled ? UPDATE_STATS_SKIPPED : UPDATE_STATS_DISABLED;
- logger.info("UPDATE STATISTICS didn't run because " + (statsEnabled ? " another UPDATE STATISTICS command was already running on the region " : " stats are disabled ")
+ rowCount = CONCURRENT_UPDATE_STATS_ROW_COUNT;
+ logger.info("UPDATE STATISTICS didn't run because another UPDATE STATISTICS command was already running on the region "
+ region.getRegionInfo().getRegionNameAsString());
}
- byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(returnValue));
+ byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
final KeyValue aggKeyValue =
KeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
@@ -729,22 +728,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
region.startRegionOperation();
boolean hasMore = false;
boolean noErrors = false;
- long rowCount = 0;
boolean compactionRunning = areStatsBeingCollectedViaCompaction();
+ long rowCount = 0;
try {
if (!compactionRunning) {
synchronized (innerScanner) {
do {
List<Cell> results = new ArrayList<Cell>();
hasMore = innerScanner.nextRaw(results);
- rowCount++;
stats.collectStatistics(results);
+ rowCount++;
compactionRunning = areStatsBeingCollectedViaCompaction();
} while (hasMore && !compactionRunning);
noErrors = true;
}
}
- return compactionRunning ? UPDATE_STATS_SKIPPED : UPDATE_STATS_RUN;
+ return compactionRunning ? COMPACTION_UPDATE_STATS_ROW_COUNT : rowCount;
} catch (IOException e) {
logger.error("IOException in update stats: " + Throwables.getStackTraceAsString(e));
throw e;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 8d1a365..fc3edbe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -122,7 +122,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
private long estimatedRows;
private long estimatedSize;
- private boolean areStatsEnabled;
static final Function<HRegionLocation, KeyRange> TO_KEY_RANGE = new Function<HRegionLocation, KeyRange>() {
@Override
@@ -370,11 +369,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return scans;
}
- // TODO: add to ResultIterators and QueryPlan interfaces?
- public boolean areStatsEnabled() {
- return this.areStatsEnabled;
- }
-
private static List<byte[]> toBoundaries(List<HRegionLocation> regionLocations) {
int nBoundaries = regionLocations.size() - 1;
List<byte[]> ranges = Lists.newArrayListWithExpectedSize(nBoundaries);
@@ -530,17 +524,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY;
List<Scan> scans = Lists.newArrayListWithExpectedSize(estGuidepostsPerRegion);
ImmutableBytesWritable guidePosts = gps.getGuidePosts();
- // If we have any guideposts, then we can definitely say that stats are enabled.
- // If we have no guideposts, though, we cannot assume that stats are disabled,
- // as the table may just be too small to have them.
- if (guidePosts.getLength() > 0) {
- areStatsEnabled = true;
- // It's possible that the server was bounced and stats have changed
- // to become enabled without a client bounce.
- this.context.getConnection().getQueryServices().setStatsEnabled(true);
- } else {
- areStatsEnabled = this.context.getConnection().getQueryServices().areStatsEnabled();
- }
ByteArrayInputStream stream = null;
DataInput input = null;
PrefixByteDecoder decoder = null;
@@ -868,18 +851,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
@Override
public void explain(List<String> planSteps) {
- ConnectionQueryServices services = context.getConnection().getQueryServices();
- boolean displayChunkCount = services.getProps().getBoolean(
+ boolean displayChunkCount = context.getConnection().getQueryServices().getProps().getBoolean(
QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB,
QueryServicesOptions.DEFAULT_EXPLAIN_CHUNK_COUNT);
- boolean displayRowCount = services.getProps().getBoolean(
- QueryServices.EXPLAIN_ROW_COUNT_ATTRIB,
- QueryServicesOptions.DEFAULT_EXPLAIN_ROW_COUNT);
StringBuilder buf = new StringBuilder();
buf.append("CLIENT ");
if (displayChunkCount) {
buf.append(this.splits.size()).append("-CHUNK ");
- if (displayRowCount && areStatsEnabled) {
+ if (estimatedRows > 0) {
buf.append(estimatedRows).append(" ROWS ");
buf.append(estimatedSize).append(" BYTES ");
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 471562a..b5f1f85 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -23,6 +23,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import co.cask.tephra.TransactionSystemClient;
+
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -44,8 +46,6 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
-import co.cask.tephra.TransactionSystemClient;
-
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
public static final int INITIAL_META_DATA_TABLE_CAPACITY = 100;
@@ -123,7 +123,4 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
TransactionSystemClient getTransactionSystemClient();
public long getRenewLeaseThresholdMilliSeconds();
public boolean isRenewingLeasesEnabled();
-
- public boolean areStatsEnabled();
- public void setStatsEnabled(boolean statsEnabled);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b428eb9..d55ab30 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -241,7 +241,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private TransactionServiceClient txServiceClient;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
- private volatile boolean areStatsEnabled;
// writes guarded by "this"
private volatile boolean closed;
@@ -1109,7 +1108,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void checkClientServerCompatibility() throws SQLException {
StringBuilder buf = new StringBuilder("The following servers require an updated " + QueryConstants.DEFAULT_COPROCESS_PATH + " to be put in the classpath of HBase: ");
boolean isIncompatible = false;
- boolean areStatsEnabled = false;
int minHBaseVersion = Integer.MAX_VALUE;
try {
List<HRegionLocation> locations = this.getAllTableRegions(SYSTEM_CATALOG_NAME_BYTES);
@@ -1143,23 +1141,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
});
for (Map.Entry<byte[],Long> result : results.entrySet()) {
// This is the "phoenix.jar" is in-place, but server is out-of-sync with client case.
- long version = result.getValue();
- // Set stats as being enabled if enabled on *any* region server (though it should
- // really match across all regions servers).
- areStatsEnabled |= MetaDataUtil.decodeStatsEnabled(version);
- if (!isCompatible(version)) {
+ if (!isCompatible(result.getValue())) {
isIncompatible = true;
HRegionLocation name = regionMap.get(result.getKey());
buf.append(name);
buf.append(';');
}
- hasIndexWALCodec &= hasIndexWALCodec(version);
- if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(version)) {
- minHBaseVersion = MetaDataUtil.decodeHBaseVersion(version);
+ hasIndexWALCodec &= hasIndexWALCodec(result.getValue());
+ if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
+ minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
}
}
lowestClusterHBaseVersion = minHBaseVersion;
- this.areStatsEnabled = areStatsEnabled;
} catch (SQLException e) {
throw e;
} catch (Throwable t) {
@@ -3386,14 +3379,4 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
- @Override
- public boolean areStatsEnabled() {
- return areStatsEnabled;
- }
-
- @Override
- public void setStatsEnabled(boolean statsEnabled) {
- this.areStatsEnabled = statsEnabled;
- }
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 453d04f..b4bbe1f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -76,7 +76,6 @@ import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.stats.PTableStats;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -107,8 +106,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private final Map<SequenceKey, SequenceInfo> sequenceMap = Maps.newHashMap();
private final String userName;
private final TransactionSystemClient txSystemClient;
- private final KeyValueBuilder kvBuilder;
- private volatile boolean areStatsEnabled;
+ private KeyValueBuilder kvBuilder;
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
@@ -138,8 +136,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
TransactionManager txnManager = new TransactionManager(config);
this.txSystemClient = new InMemoryTxSystemClient(txnManager);
- // Just check the properties on the client side (instead of normally the server side)
- this.areStatsEnabled = StatisticsUtil.isStatsEnabled(config);
}
private PMetaData newEmptyMetaData() {
@@ -531,7 +527,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return txSystemClient;
}
- @Override
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return new MetaDataMutationResult(MutationCode.FUNCTION_NOT_FOUND, 0l, null);
@@ -583,7 +578,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return false;
}
- @Override
public HRegionLocation getTableRegionLocation(byte[] tableName, byte[] row) throws SQLException {
List<HRegionLocation> regions = tableSplits.get(Bytes.toString(tableName));
if (regions != null) {
@@ -598,14 +592,4 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
new HRegionInfo(TableName.valueOf(tableName), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW),
SERVER_NAME, -1);
}
-
- @Override
- public boolean areStatsEnabled() {
- return areStatsEnabled;
- }
-
- @Override
- public void setStatsEnabled(boolean statsEnabled) {
- this.areStatsEnabled = statsEnabled;
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 2d0f677..4c7446b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -310,14 +310,4 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
throws SQLException {
return getDelegate().getTableRegionLocation(tableName, row);
}
-
- @Override
- public boolean areStatsEnabled() {
- return getDelegate().areStatsEnabled();
- }
-
- @Override
- public void setStatsEnabled(boolean statsEnabled) {
- getDelegate().setStatsEnabled(statsEnabled);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 32b4fc0..1efcd8c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -162,7 +162,6 @@ public interface QueryServices extends SQLCloseable {
public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets";
public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority";
public static final String EXPLAIN_CHUNK_COUNT_ATTRIB = "phoenix.explain.displayChunkCount";
- public static final String EXPLAIN_ROW_COUNT_ATTRIB = "phoenix.explain.displayRowCount";
public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 3e0ffe1..27c5693 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -29,7 +29,6 @@ import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB
import static org.apache.phoenix.query.QueryServices.DELAY_FOR_SCHEMA_UPDATE_CHECK;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.EXPLAIN_ROW_COUNT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GLOBAL_METRICS_ENABLED;
@@ -197,7 +196,6 @@ public class QueryServicesOptions {
*/
public static final int DEFAULT_COPROCESSOR_PRIORITY = Coprocessor.PRIORITY_SYSTEM/2 + Coprocessor.PRIORITY_USER/2; // Divide individually to prevent any overflow
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = true;
- public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = true;
public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
@@ -575,11 +573,6 @@ public class QueryServicesOptions {
return this;
}
- public QueryServicesOptions setExplainRowCount(boolean showRowCount) {
- config.setBoolean(EXPLAIN_ROW_COUNT_ATTRIB, showRowCount);
- return this;
- }
-
public QueryServicesOptions setAllowOnlineSchemaUpdate(boolean allow) {
config.setBoolean(ALLOW_ONLINE_TABLE_SCHEMA_UPDATE, allow);
return this;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index eee42ea..7f3f850 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -182,7 +182,6 @@ import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.stats.PTableStats;
-import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
@@ -956,7 +955,6 @@ public class MetaDataClient {
}
MutationState mutationState = plan.execute();
rowCount = mutationState.getUpdateCount();
- this.getConnection().getQueryServices().setStatsEnabled(rowCount != StatisticsCollectionRunTracker.UPDATE_STATS_DISABLED);
}
/*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index 6c73f16..1063229 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -21,10 +21,13 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
index 6413e37..4ed3325 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -44,9 +45,9 @@ public class StatisticsCollectionRunTracker {
.newSetFromMap(new ConcurrentHashMap<HRegionInfo, Boolean>());
private final ExecutorService executor;
- public static final long UPDATE_STATS_RUN = 1L;
- public static final long UPDATE_STATS_SKIPPED = 100000L;
- public static final long UPDATE_STATS_DISABLED = 0;
+ // Constants added for testing purposes
+ public static final long CONCURRENT_UPDATE_STATS_ROW_COUNT = -100l;
+ public static final long COMPACTION_UPDATE_STATS_ROW_COUNT = -200l;
public static StatisticsCollectionRunTracker getInstance(Configuration config) {
StatisticsCollectionRunTracker result = INSTANCE;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 9dd72e8..5b47104 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
@@ -41,8 +40,6 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
@@ -223,17 +220,4 @@ public class StatisticsUtil {
ptr.set(row, gpOffset, row.length - gpOffset);
return ByteUtil.copyKeyBytesIfNecessary(ptr);
}
-
- public static boolean isStatsEnabled(Configuration conf) {
- if (conf.getBoolean(QueryServices.STATS_ENABLED_ATTRIB, true)) {
- if (conf.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) != 1) {
- if (conf.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) < HConstants.DEFAULT_MAX_FILE_SIZE) {
- return true;
- }
- }
- }
- return false;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 5ee3c0a..6d5903d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -57,7 +56,6 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PLong;
@@ -105,25 +103,26 @@ public class MetaDataUtil {
// The second byte in int would be the major version, 3rd byte minor version, and 4th byte
// patch version.
public static int decodePhoenixVersion(long version) {
- return (int) ((version << Byte.SIZE * 4) >>> Byte.SIZE * 5);
+ return (int) ((version << Byte.SIZE * 3) >>> Byte.SIZE * 4);
+ }
+
+ // TODO: generalize this to use two bytes to return a SQL error code instead
+ public static long encodeHasIndexWALCodec(long version, boolean isValid) {
+ if (!isValid) {
+ return version | 1;
+ }
+ return version;
}
public static boolean decodeHasIndexWALCodec(long version) {
return (version & 0xF) == 0;
}
- // Given the encoded integer representing the phoenix version in the encoded version value.
- // The second byte in int would be the major version, 3rd byte minor version, and 4th byte
- // patch version.
- public static boolean decodeStatsEnabled(long version) {
- return ((int) ((version << Byte.SIZE * 3) >>> Byte.SIZE * 7) & 0x1) != 0;
- }
-
// Given the encoded integer representing the client hbase version in the encoded version value.
// The second byte in int would be the major version, 3rd byte minor version, and 4th byte
// patch version.
public static int decodeHBaseVersion(long version) {
- return (int) (version >>> (Byte.SIZE * 5));
+ return (int) (version >>> Byte.SIZE * 5);
}
public static String decodeHBaseVersionAsString(int version) {
@@ -133,33 +132,17 @@ public class MetaDataUtil {
return major + "." + minor + "." + patch;
}
- // The first 3 bytes of the long is used to encoding the HBase version as major.minor.patch.
- // The next 4 bytes of the value is used to encode the Phoenix version as major.minor.patch.
- /**
- * Encode HBase and Phoenix version along with some server-side config information such
- * as whether WAL codec is installed (necessary for non transactional, mutable secondar
- * indexing), and whether stats are enabled.
- * @param env RegionCoprocessorEnvironment to access HBase version and Configuration.
- * @return long value sent back during initialization of a cluster connection.
- */
- public static long encodeVersion(String hbaseVersionStr, Configuration config) {
- long hbaseVersion = VersionUtil.encodeVersion(hbaseVersionStr);
- long statsEnabled = StatisticsUtil.isStatsEnabled(config) ? 1 : 0;
- long phoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ public static int encodePhoenixVersion() {
+ return VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
MetaDataProtocol.PHOENIX_PATCH_NUMBER);
- long walCodec = IndexManagementUtil.isWALEditCodecSet(config) ? 0 : 1;
- long version =
- // Encode HBase major, minor, patch version
- (hbaseVersion << (Byte.SIZE * 5))
- // Encode if stats are enabled on the server side
- | (statsEnabled << (Byte.SIZE * 4))
- // Encode Phoenix major, minor, patch version
- | (phoenixVersion << (Byte.SIZE * 1))
- // Encode whether or not non transactional, mutable secondary indexing was configured properly.
- | walCodec;
- return version;
}
-
+
+ public static long encodeHBaseAndPhoenixVersions(String hbaseVersion) {
+ return (((long) VersionUtil.encodeVersion(hbaseVersion)) << (Byte.SIZE * 5)) |
+ (((long) VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ MetaDataProtocol.PHOENIX_PATCH_NUMBER)) << (Byte.SIZE * 1));
+ }
+
public static void getTenantIdAndSchemaAndTableName(List<Mutation> tableMetadata, byte[][] rowKeyMetaData) {
Mutation m = getTableHeaderRow(tableMetadata);
getVarChars(m.getRow(), 3, rowKeyMetaData);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 6ae655c..29a7001 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -54,7 +54,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = false; // TODO: update explain plans in test and set to true
- public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = false; // TODO: update explain plans in test and set to true
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
@@ -81,7 +80,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static QueryServicesOptions getDefaultServicesOptions() {
return withDefaults()
.setExplainChunkCount(DEFAULT_EXPLAIN_CHUNK_COUNT)
- .setExplainRowCount(DEFAULT_EXPLAIN_ROW_COUNT)
.setSequenceSaltBuckets(DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS)
.setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
index 7ffc054..1e06379 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
@@ -21,19 +21,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.query.HBaseFactoryProvider;
-import org.apache.phoenix.query.QueryServices;
import org.junit.Test;
@@ -62,35 +58,6 @@ public class MetaDataUtilTest {
assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 3, 5));
}
- @Test
- public void testEncodeDecode() {
- String hbaseVersionStr = "0.98.14";
- Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
- config.setBoolean(QueryServices.STATS_ENABLED_ATTRIB, false);
-
- long version = MetaDataUtil.encodeVersion(hbaseVersionStr, config);
- int hbaseVersion = MetaDataUtil.decodeHBaseVersion(version);
- int expectedHBaseVersion = VersionUtil.encodeVersion(0, 98, 14);
- assertEquals(expectedHBaseVersion, hbaseVersion);
- boolean areStatsEnabled = MetaDataUtil.decodeStatsEnabled(version);
- assertFalse(areStatsEnabled);
- int phoenixVersion = MetaDataUtil.decodePhoenixVersion(version);
- int expectedPhoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
- MetaDataProtocol.PHOENIX_PATCH_NUMBER);
- assertEquals(expectedPhoenixVersion, phoenixVersion);
-
- config.setBoolean(QueryServices.STATS_ENABLED_ATTRIB, true);
- version = MetaDataUtil.encodeVersion(hbaseVersionStr, config);
- hbaseVersion = MetaDataUtil.decodeHBaseVersion(version);
- expectedHBaseVersion = VersionUtil.encodeVersion(0, 98, 14);
- assertEquals(expectedHBaseVersion, hbaseVersion);
- areStatsEnabled = MetaDataUtil.decodeStatsEnabled(version);
- assertTrue(areStatsEnabled);
- phoenixVersion = MetaDataUtil.decodePhoenixVersion(version);
- expectedPhoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
- MetaDataProtocol.PHOENIX_PATCH_NUMBER);
- assertEquals(expectedPhoenixVersion, phoenixVersion);
- }
/**
* Ensure it supports {@link GenericKeyValueBuilder}
* @throws Exception on failure
[2/2] phoenix git commit: PHOENIX-2707 Differentiate between a
table+family have zero guideposts from not having collected guideposts
Posted by ja...@apache.org.
PHOENIX-2707 Differentiate between a table+family have zero guideposts from not having collected guideposts
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c25f8879
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c25f8879
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c25f8879
Branch: refs/heads/master
Commit: c25f88791c86cd65ea21810406eec199403d78aa
Parents: dbc9ee9
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Feb 23 14:53:11 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Feb 23 14:53:11 2016 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/IndexToolIT.java | 3 +-
.../phoenix/end2end/MutableIndexToolIT.java | 1 +
.../end2end/StatsCollectionDisabledIT.java | 21 ++-
.../phoenix/end2end/StatsCollectorIT.java | 57 +++++++-
.../phoenix/iterate/BaseResultIterators.java | 140 +++++++++++--------
.../apache/phoenix/query/QueryConstants.java | 2 +-
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 7 +
.../stats/DefaultStatisticsCollector.java | 33 ++---
.../phoenix/schema/stats/GuidePostsInfo.java | 2 +-
.../schema/stats/GuidePostsInfoBuilder.java | 6 -
.../phoenix/schema/stats/PTableStatsImpl.java | 5 +-
.../phoenix/schema/stats/StatisticsUtil.java | 23 +--
.../phoenix/schema/stats/StatisticsWriter.java | 60 ++++----
.../apache/phoenix/util/PrefixByteCodec.java | 37 ++---
.../phoenix/query/QueryServicesTestImpl.java | 2 +
16 files changed, 246 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index aba9c11..fe95470 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -108,7 +108,8 @@ public class IndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String fullTableName = SchemaUtil.getTableName(schemaName, dataTable);
final String indxTable = String.format("%s_%s", dataTable, "INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.TRANSACTIONS_ENABLED, "true");
+ props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
index 0791479..8125007 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
@@ -61,6 +61,7 @@ public class MutableIndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String dataTable = "DATA_TABLE5";
final String indxTable = String.format("%s_%s",dataTable,"INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
index a92a665..54ffa7c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
@@ -17,24 +17,26 @@
*/
package org.apache.phoenix.end2end;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.Properties;
-import com.google.common.collect.Maps;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertFalse;
+import com.google.common.collect.Maps;
/**
* Verifies that statistics are not collected if they are disabled via a setting
@@ -43,10 +45,12 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(5);
// Must update config before starting server
props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
props.put(QueryServices.STATS_ENABLED_ATTRIB, Boolean.toString(false));
+ props.put(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
+ props.put(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString());
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -65,6 +69,11 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
assertFalse(rs.next());
rs.close();
stmt.close();
- conn.close();
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM T1");
+ String explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER T1",
+ explainPlan);
+ conn.close();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index e72f41f..bc575fd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -44,6 +44,7 @@ import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
@@ -65,10 +66,11 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(10);
// Must update config before starting server
props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
props.put(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
+ props.put(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString());
props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(1024));
props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
@@ -86,6 +88,55 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
}
@Test
+ public void testUpdateEmptyStats() throws Exception {
+ Connection conn;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName +" ( k CHAR(1) PRIMARY KEY )" + tableDDLOptions);
+ conn.createStatement().execute("UPDATE STATISTICS " + tableName);
+ ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName);
+ String explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 1-CHUNK 0 ROWS 0 BYTES PARALLEL 1-WAY FULL SCAN OVER " + fullTableName + "\n" +
+ " SERVER FILTER BY FIRST KEY ONLY",
+ explainPlan);
+ conn.close();
+ }
+
+ @Test
+ public void testSomeUpdateEmptyStats() throws Exception {
+ Connection conn;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String fullTableName = this.fullTableName + "_SALTED";
+ // props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName +" ( k VARCHAR PRIMARY KEY, a.v1 VARCHAR, b.v2 VARCHAR ) " + tableDDLOptions + (tableDDLOptions.isEmpty() ? "" : ",") + "SALT_BUCKETS = 3");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + "(k,v1) VALUES('a','123456789')");
+ conn.createStatement().execute("UPDATE STATISTICS " + fullTableName);
+ ResultSet rs;
+ String explainPlan;
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT v2 FROM " + fullTableName + " WHERE v2='foo'");
+ explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 3-CHUNK 0 ROWS 0 BYTES PARALLEL 3-WAY FULL SCAN OVER " + fullTableName + "\n" +
+ " SERVER FILTER BY B.V2 = 'foo'\n" +
+ "CLIENT MERGE SORT",
+ explainPlan);
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName);
+ explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 4-CHUNK 1 ROWS 34 BYTES PARALLEL 3-WAY FULL SCAN OVER " + fullTableName + "\n" +
+ "CLIENT MERGE SORT",
+ explainPlan);
+
+ conn.close();
+ }
+
+ @Test
public void testUpdateStats() throws SQLException, IOException,
InterruptedException {
Connection conn;
@@ -336,7 +387,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
// and forcing the new stats to be pulled over.
int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
- assertEquals(0, rowCount);
+ assertEquals(10, rowCount);
}
List<KeyRange>keyRanges = getAllSplits(conn, tableName);
assertEquals(nRows+1, keyRanges.size());
@@ -356,7 +407,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
// and force us to pull over the new stats
int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
- assertEquals(0, rowCount);
+ assertEquals(5, rowCount);
keyRanges = getAllSplits(conn, tableName);
}
assertEquals(nRows/2+1, keyRanges.size());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index fc3edbe..a48de13 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -95,6 +95,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.io.Closeables;
/**
@@ -120,8 +121,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final ParallelScanGrouper scanGrouper;
// TODO: too much nesting here - breakup into new classes.
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
- private long estimatedRows;
- private long estimatedSize;
+ private Long estimatedRows;
+ private Long estimatedSize;
+ private boolean hasGuidePosts;
static final Function<HRegionLocation, KeyRange> TO_KEY_RANGE = new Function<HRegionLocation, KeyRange>() {
@Override
@@ -401,7 +403,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
* be further parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan
* entire regions worth of data to track where to put the guide posts.
*/
- if (!useStats()) { return GuidePostsInfo.EMPTY_GUIDEPOST; }
+ if (!useStats()) { return GuidePostsInfo.NO_GUIDEPOST; }
GuidePostsInfo gps = null;
PTable table = getTable();
@@ -424,7 +426,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
}
}
- if (gps == null) { return GuidePostsInfo.EMPTY_GUIDEPOST; }
+ if (gps == null) { return GuidePostsInfo.NO_GUIDEPOST; }
return gps;
}
@@ -490,6 +492,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
}
GuidePostsInfo gps = getGuidePosts(whereConditions);
+ hasGuidePosts = gps != GuidePostsInfo.NO_GUIDEPOST;
boolean traverseAllRegions = isSalted || isLocalIndex;
if (!traverseAllRegions) {
byte[] scanStartRow = scan.getStartRow();
@@ -528,63 +531,75 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
DataInput input = null;
PrefixByteDecoder decoder = null;
int guideIndex = 0;
- if (gpsSize > 0) {
- stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength());
- input = new DataInputStream(stream);
- decoder = new PrefixByteDecoder(gps.getMaxLength());
- try {
- while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0
- && currentKey.getLength() != 0) {
- guideIndex++;
- }
- } catch (EOFException e) {}
- }
- byte[] currentKeyBytes = currentKey.copyBytes();
-
- // Merge bisect with guideposts for all but the last region
- while (regionIndex <= stopIndex) {
- byte[] currentGuidePostBytes = currentGuidePost.copyBytes();
- byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY;
- if (regionIndex == stopIndex) {
- endKey = stopKey;
- } else {
- endKey = regionBoundaries.get(regionIndex);
- }
- HRegionLocation regionLocation = regionLocations.get(regionIndex);
- if (isLocalIndex) {
- HRegionInfo regionInfo = regionLocation.getRegionInfo();
- endRegionKey = regionInfo.getEndKey();
- keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey);
+ long estimatedRows = 0;
+ long estimatedSize = 0;
+ try {
+ if (gpsSize > 0) {
+ stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength());
+ input = new DataInputStream(stream);
+ decoder = new PrefixByteDecoder(gps.getMaxLength());
+ try {
+ while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0
+ && currentKey.getLength() != 0) {
+ guideIndex++;
+ }
+ } catch (EOFException e) {}
}
- try {
- while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
- Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
- false);
- estimatedRows += gps.getRowCounts().get(guideIndex);
- estimatedSize += gps.getByteCounts().get(guideIndex);
- scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
- currentKeyBytes = currentGuidePost.copyBytes();
- currentGuidePost = PrefixByteCodec.decode(decoder, input);
- currentGuidePostBytes = currentGuidePost.copyBytes();
- guideIndex++;
+ byte[] currentKeyBytes = currentKey.copyBytes();
+
+ // Merge bisect with guideposts for all but the last region
+ while (regionIndex <= stopIndex) {
+ byte[] currentGuidePostBytes = currentGuidePost.copyBytes();
+ byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY;
+ if (regionIndex == stopIndex) {
+ endKey = stopKey;
+ } else {
+ endKey = regionBoundaries.get(regionIndex);
}
- } catch (EOFException e) {}
- Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true);
- if (isLocalIndex) {
- if (newScan != null) {
- newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
- } else if (!scans.isEmpty()) {
- scans.get(scans.size()-1).setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
+ HRegionLocation regionLocation = regionLocations.get(regionIndex);
+ if (isLocalIndex) {
+ HRegionInfo regionInfo = regionLocation.getRegionInfo();
+ endRegionKey = regionInfo.getEndKey();
+ keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey);
}
+ try {
+ while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
+ Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
+ false);
+ estimatedRows += gps.getRowCounts().get(guideIndex);
+ estimatedSize += gps.getByteCounts().get(guideIndex);
+ scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
+ currentKeyBytes = currentGuidePost.copyBytes();
+ currentGuidePost = PrefixByteCodec.decode(decoder, input);
+ currentGuidePostBytes = currentGuidePost.copyBytes();
+ guideIndex++;
+ }
+ } catch (EOFException e) {}
+ Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true);
+ if (isLocalIndex) {
+ if (newScan != null) {
+ newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
+ } else if (!scans.isEmpty()) {
+ scans.get(scans.size()-1).setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
+ }
+ }
+ scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation);
+ currentKeyBytes = endKey;
+ regionIndex++;
}
- scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation);
- currentKeyBytes = endKey;
- regionIndex++;
- }
- if (!scans.isEmpty()) { // Add any remaining scans
- parallelScans.add(scans);
+ if (hasGuidePosts) {
+ this.estimatedRows = estimatedRows;
+ this.estimatedSize = estimatedSize;
+ } else {
+ this.estimatedRows = null;
+ this.estimatedSize = null;
+ }
+ if (!scans.isEmpty()) { // Add any remaining scans
+ parallelScans.add(scans);
+ }
+ } finally {
+ if (stream != null) Closeables.closeQuietly(stream);
}
- PrefixByteCodec.close(stream);
return parallelScans;
}
@@ -857,8 +872,11 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
StringBuilder buf = new StringBuilder();
buf.append("CLIENT ");
if (displayChunkCount) {
+ boolean displayRowCount = context.getConnection().getQueryServices().getProps().getBoolean(
+ QueryServices.EXPLAIN_ROW_COUNT_ATTRIB,
+ QueryServicesOptions.DEFAULT_EXPLAIN_ROW_COUNT);
buf.append(this.splits.size()).append("-CHUNK ");
- if (estimatedRows > 0) {
+ if (displayRowCount && hasGuidePosts) {
buf.append(estimatedRows).append(" ROWS ");
buf.append(estimatedSize).append(" BYTES ");
}
@@ -867,6 +885,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
explain(buf.toString(),planSteps);
}
+ public Long getEstimatedRowCount() {
+ return this.estimatedRows;
+ }
+
+ public Long getEstimatedByteCount() {
+ return this.estimatedSize;
+ }
+
@Override
public String toString() {
return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 63d4e07..471ac73 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -271,7 +271,7 @@ public interface QueryConstants {
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
// PK columns
PHYSICAL_NAME + " VARCHAR NOT NULL," +
- COLUMN_FAMILY + " VARCHAR," +
+ COLUMN_FAMILY + " VARCHAR NOT NULL," +
GUIDE_POST_KEY + " VARBINARY," +
GUIDE_POSTS_WIDTH + " BIGINT," +
LAST_STATS_UPDATE_TIME+ " DATE, "+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 1efcd8c..32b4fc0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -162,6 +162,7 @@ public interface QueryServices extends SQLCloseable {
public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets";
public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority";
public static final String EXPLAIN_CHUNK_COUNT_ATTRIB = "phoenix.explain.displayChunkCount";
+ public static final String EXPLAIN_ROW_COUNT_ATTRIB = "phoenix.explain.displayRowCount";
public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 27c5693..3e0ffe1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -29,6 +29,7 @@ import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB
import static org.apache.phoenix.query.QueryServices.DELAY_FOR_SCHEMA_UPDATE_CHECK;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.EXPLAIN_ROW_COUNT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GLOBAL_METRICS_ENABLED;
@@ -196,6 +197,7 @@ public class QueryServicesOptions {
*/
public static final int DEFAULT_COPROCESSOR_PRIORITY = Coprocessor.PRIORITY_SYSTEM/2 + Coprocessor.PRIORITY_USER/2; // Divide individually to prevent any overflow
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = true;
+ public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = true;
public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
@@ -573,6 +575,11 @@ public class QueryServicesOptions {
return this;
}
+ public QueryServicesOptions setExplainRowCount(boolean showRowCount) {
+ config.setBoolean(EXPLAIN_ROW_COUNT_ATTRIB, showRowCount);
+ return this;
+ }
+
public QueryServicesOptions setAllowOnlineSchemaUpdate(boolean allow) {
config.setBoolean(ALLOW_ONLINE_TABLE_SCHEMA_UPDATE, allow);
return this;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index cb6f5d4..b81d206 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -24,13 +24,11 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
@@ -50,18 +48,15 @@ import com.google.common.collect.Maps;
/**
* A default implementation of the Statistics tracker that helps to collect stats like min key, max key and guideposts.
- * TODO: review timestamps used for stats. We support the user controlling the timestamps, so we should honor that with
- * timestamps for stats as well. The issue is for compaction, though. I don't know of a way for the user to specify any
- * timestamp for that. Perhaps best to use current time across the board for now.
*/
class DefaultStatisticsCollector implements StatisticsCollector {
private static final Logger logger = LoggerFactory.getLogger(DefaultStatisticsCollector.class);
+ private final Map<ImmutableBytesPtr, Pair<Long, GuidePostsInfoBuilder>> guidePostsInfoWriterMap = Maps.newHashMap();
+ private final StatisticsWriter statsWriter;
+ private final Pair<Long, GuidePostsInfoBuilder> cachedGps;
private long guidepostDepth;
private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
- private Map<ImmutableBytesPtr, Pair<Long, GuidePostsInfoBuilder>> guidePostsInfoWriterMap = Maps.newHashMap();
- protected StatisticsWriter statsTable;
- private Pair<Long, GuidePostsInfoBuilder> cachedGps = null;
DefaultStatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family,
byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
@@ -87,12 +82,14 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
// Get the stats table associated with the current table on which the CP is
// triggered
- this.statsTable = StatisticsWriter.newWriter(env, tableName, clientTimeStamp);
+ this.statsWriter = StatisticsWriter.newWriter(env, tableName, clientTimeStamp);
// in a compaction we know the one family ahead of time
if (family != null) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family);
cachedGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
guidePostsInfoWriterMap.put(cfKey, cachedGps);
+ } else {
+ cachedGps = null;
}
}
@@ -103,7 +100,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
@Override
public void close() throws IOException {
- this.statsTable.close();
+ this.statsWriter.close();
}
@Override
@@ -129,9 +126,13 @@ class DefaultStatisticsCollector implements StatisticsCollector {
// Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS
// This will not impact a stats collection of single column family during compaction as
// guidePostsInfoWriterMap cannot be empty in this case.
- if (guidePostsInfoWriterMap.keySet().isEmpty()) {
+ if (cachedGps == null) {
for (Store store : region.getStores()) {
- statsTable.deleteStats(region, this, new ImmutableBytesPtr(store.getFamily().getName()), mutations);
+ ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
+ if (!guidePostsInfoWriterMap.containsKey(cfKey)) {
+ Pair<Long, GuidePostsInfoBuilder> emptyGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
+ guidePostsInfoWriterMap.put(cfKey, emptyGps);
+ }
}
}
for (ImmutableBytesPtr fam : guidePostsInfoWriterMap.keySet()) {
@@ -139,12 +140,12 @@ class DefaultStatisticsCollector implements StatisticsCollector {
if (logger.isDebugEnabled()) {
logger.debug("Deleting the stats for the region " + region.getRegionInfo());
}
- statsTable.deleteStats(region, this, fam, mutations);
+ statsWriter.deleteStats(region, this, fam, mutations);
}
if (logger.isDebugEnabled()) {
logger.debug("Adding new stats for the region " + region.getRegionInfo());
}
- statsTable.addStats(this, fam, mutations);
+ statsWriter.addStats(this, fam, mutations);
}
} catch (IOException e) {
logger.error("Failed to update statistics table!", e);
@@ -153,7 +154,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
private void commitStats(List<Mutation> mutations) throws IOException {
- statsTable.commitStats(mutations);
+ statsWriter.commitStats(mutations);
}
/**
@@ -212,7 +213,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
protected InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, InternalScanner internalScan,
ImmutableBytesPtr family) {
- return new StatisticsScanner(this, statsTable, env, internalScan, family);
+ return new StatisticsScanner(this, statsWriter, env, internalScan, family);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
index b8cc3f1..291cb6d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
@@ -38,7 +38,7 @@ public class GuidePostsInfo {
*/
private int maxLength;
- public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
+ public final static GuidePostsInfo NO_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0);
public int getMaxLength() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
index 2133349..10d59ee 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteEncoder;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
@@ -107,15 +106,10 @@ public class GuidePostsInfoBuilder {
return addGuidePosts(new ImmutableBytesWritable(row), byteCount, rowCount);
}
- private void close() {
- PrefixByteCodec.close(stream);
- }
-
public GuidePostsInfo build() {
this.guidePosts.set(stream.getBuffer(), 0, stream.size());
GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCounts, this.guidePosts, this.rowCounts,
this.maxLength, this.guidePostsCount);
- this.close();
return guidePostsInfo;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
index dacc213..42e7bed 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
@@ -32,6 +32,7 @@ import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteDecoder;
import org.apache.phoenix.util.SizedUtil;
+import com.google.common.io.Closeables;
import com.sun.istack.NotNull;
/**
@@ -89,13 +90,13 @@ public class PTableStatsImpl implements PTableStats {
} catch (EOFException e) { // Ignore as this signifies we're done
} finally {
- PrefixByteCodec.close(stream);
+ Closeables.closeQuietly(stream);
}
buf.setLength(buf.length() - 1);
}
buf.append(")");
} finally {
- PrefixByteCodec.close(stream);
+ Closeables.closeQuietly(stream);
}
}
buf.append("]");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 5b47104..5e03be5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -62,15 +62,19 @@ public class StatisticsUtil {
public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) {
// always starts with the source table
- byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.getLength() + 2];
+ int guidePostLength = guidePostStartKey.getLength();
+ boolean hasGuidePost = guidePostLength > 0;
+ byte[] rowKey = new byte[table.length + fam.getLength() + guidePostLength + (hasGuidePost ? 2 : 1)];
int offset = 0;
System.arraycopy(table, 0, rowKey, offset, table.length);
offset += table.length;
rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength());
- offset += fam.getLength();
- rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
- System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostStartKey.getLength());
+ if (hasGuidePost) {
+ offset += fam.getLength();
+ rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
+ System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostLength);
+ }
return rowKey;
}
@@ -215,9 +219,12 @@ public class StatisticsUtil {
}
public static byte[] getGuidePostsInfoFromRowKey(byte[] tableNameBytes, byte[] fam, byte[] row) {
- ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- int gpOffset = tableNameBytes.length + 1 + fam.length + 1;
- ptr.set(row, gpOffset, row.length - gpOffset);
- return ByteUtil.copyKeyBytesIfNecessary(ptr);
+ if (row.length > tableNameBytes.length + 1 + fam.length) {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ int gpOffset = tableNameBytes.length + 1 + fam.length + 1;
+ ptr.set(row, gpOffset, row.length - gpOffset);
+ return ByteUtil.copyKeyBytesIfNecessary(ptr);
+ }
+ return ByteUtil.EMPTY_BYTE_ARRAY;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index f8c888d..a727c1c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -49,7 +49,6 @@ import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteDecoder;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TimeKeeper;
@@ -143,33 +142,46 @@ public class StatisticsWriter implements Closeable {
List<Long> byteCounts = gps.getByteCounts();
List<Long> rowCounts = gps.getRowCounts();
ImmutableBytesWritable keys = gps.getGuidePosts();
- ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength());
- DataInput input = new DataInputStream(stream);
- PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength());
- int guidePostCount = 0;
- try {
- while (true) {
- ImmutableBytesWritable ptr = decoder.decode(input);
- byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr);
- Put put = new Put(prefix);
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
- timeStamp, PLong.INSTANCE.toBytes(byteCounts.get(guidePostCount)));
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
- PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
- PLong.INSTANCE.toBytes(rowCounts.get(guidePostCount)));
- // Add our empty column value so queries behave correctly
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
- ByteUtil.EMPTY_BYTE_ARRAY);
- mutations.add(put);
- guidePostCount++;
- }
- } catch (EOFException e) { // Ignore as this signifies we're done
+ boolean hasGuidePosts = keys.getLength() > 0;
+ if (hasGuidePosts) {
+ int guidePostCount = 0;
+ try (ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength())) {
+ DataInput input = new DataInputStream(stream);
+ PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength());
+ do {
+ ImmutableBytesWritable ptr = decoder.decode(input);
+ addGuidepost(cfKey, mutations, ptr, byteCounts.get(guidePostCount), rowCounts.get(guidePostCount), timeStamp);
+ guidePostCount++;
+ } while (decoder != null);
+ } catch (EOFException e) { // Ignore as this signifies we're done
- } finally {
- PrefixByteCodec.close(stream);
+ }
+ // If we've written guideposts with a guidepost key, then delete the
+ // empty guidepost indicator that may have been written by other
+ // regions.
+ byte[] rowKey = StatisticsUtil.getRowKey(tableName, cfKey, ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY);
+ Delete delete = new Delete(rowKey, timeStamp);
+ mutations.add(delete);
+ } else {
+ addGuidepost(cfKey, mutations, ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY, 0, 0, timeStamp);
}
}
}
+
+ @SuppressWarnings("deprecation")
+ private void addGuidepost(ImmutableBytesPtr cfKey, List<Mutation> mutations, ImmutableBytesWritable ptr, long byteCount, long rowCount, long timeStamp) {
+ byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr);
+ Put put = new Put(prefix);
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
+ timeStamp, PLong.INSTANCE.toBytes(byteCount));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
+ PLong.INSTANCE.toBytes(rowCount));
+ // Add our empty column value so queries behave correctly
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
+ ByteUtil.EMPTY_BYTE_ARRAY);
+ mutations.add(put);
+ }
private static MutationType getMutationType(Mutation m) throws IOException {
if (m instanceof Put) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
index 8c3aa80..2641b6a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
@@ -18,7 +18,6 @@
package org.apache.phoenix.util;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
@@ -52,15 +51,15 @@ public class PrefixByteCodec {
}
public static int encodeBytes(List<byte[]> listOfBytes, ImmutableBytesWritable ptr) throws IOException {
- TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes));
- DataOutput output = new DataOutputStream(stream);
- PrefixByteEncoder encoder = new PrefixByteEncoder();
- for (byte[] bytes : listOfBytes) {
- encoder.encode(output, bytes, 0, bytes.length);
+ try (TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes))) {
+ DataOutput output = new DataOutputStream(stream);
+ PrefixByteEncoder encoder = new PrefixByteEncoder();
+ for (byte[] bytes : listOfBytes) {
+ encoder.encode(output, bytes, 0, bytes.length);
+ }
+ ptr.set(stream.getBuffer(), 0, stream.size());
+ return encoder.getMaxLength();
}
- close(stream);
- ptr.set(stream.getBuffer(), 0, stream.size());
- return encoder.getMaxLength();
}
public static int calculateSize(List<byte[]> listOfBytes) {
@@ -81,24 +80,4 @@ public class PrefixByteCodec {
throw new RuntimeException(e);
}
}
-
- public static void close(ByteArrayInputStream stream) {
- if (stream != null) {
- try {
- stream.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public static void close(ByteArrayOutputStream stream) {
- if (stream != null) {
- try {
- stream.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 29a7001..6ae655c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -54,6 +54,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = false; // TODO: update explain plans in test and set to true
+ public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = false; // TODO: update explain plans in test and set to true
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
@@ -80,6 +81,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static QueryServicesOptions getDefaultServicesOptions() {
return withDefaults()
.setExplainChunkCount(DEFAULT_EXPLAIN_CHUNK_COUNT)
+ .setExplainRowCount(DEFAULT_EXPLAIN_ROW_COUNT)
.setSequenceSaltBuckets(DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS)
.setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE)