You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2016/02/25 21:39:04 UTC
[01/12] phoenix git commit: PHOENIX-2706 Implement client-side
mechanism to know if stats are enabled
Repository: phoenix
Updated Branches:
refs/heads/calcite 9ac854ae5 -> 36ee23ab7
PHOENIX-2706 Implement client-side mechanism to know if stats are enabled
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a1e6ae44
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a1e6ae44
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a1e6ae44
Branch: refs/heads/calcite
Commit: a1e6ae44be757b3fc5c4192ad55ecd8f3c03c01a
Parents: e4acd0c
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Feb 22 18:49:04 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Feb 22 18:49:04 2016 -0800
----------------------------------------------------------------------
.../StatisticsCollectionRunTrackerIT.java | 10 ++--
.../org/apache/phoenix/end2end/IndexToolIT.java | 3 +-
.../phoenix/end2end/MutableIndexToolIT.java | 1 +
.../end2end/StatsCollectionDisabledIT.java | 19 +++++--
.../phoenix/end2end/StatsCollectorIT.java | 3 ++
.../coprocessor/MetaDataEndpointImpl.java | 12 +----
.../UngroupedAggregateRegionObserver.java | 27 +++++-----
.../phoenix/iterate/BaseResultIterators.java | 25 ++++++++-
.../phoenix/query/ConnectionQueryServices.java | 7 ++-
.../query/ConnectionQueryServicesImpl.java | 25 +++++++--
.../query/ConnectionlessQueryServicesImpl.java | 18 ++++++-
.../query/DelegateConnectionQueryServices.java | 10 ++++
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 7 +++
.../apache/phoenix/schema/MetaDataClient.java | 2 +
.../schema/stats/NoOpStatisticsCollector.java | 3 --
.../stats/StatisticsCollectionRunTracker.java | 7 ++-
.../phoenix/schema/stats/StatisticsUtil.java | 16 ++++++
.../org/apache/phoenix/util/MetaDataUtil.java | 55 +++++++++++++-------
.../phoenix/query/QueryServicesTestImpl.java | 2 +
.../apache/phoenix/util/MetaDataUtilTest.java | 33 ++++++++++++
21 files changed, 215 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
index bf567f0..345400e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
@@ -17,8 +17,7 @@
*/
package org.apache.phoenix.coprocessor;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_SKIPPED;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -38,7 +37,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.util.ReadOnlyProps;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -105,7 +103,8 @@ public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManaged
HRegionInfo regionInfo = createTableAndGetRegion(tableName);
// simulate stats collection via major compaction by marking the region as compacting in the tracker
markRegionAsCompacting(regionInfo);
- Assert.assertEquals("Row count didn't match", COMPACTION_UPDATE_STATS_ROW_COUNT, runUpdateStats(tableName));
+ long returnValue = runUpdateStats(tableName);
+ assertTrue("Update stats should have been skipped", returnValue >= UPDATE_STATS_SKIPPED);
StatisticsCollectionRunTracker tracker =
StatisticsCollectionRunTracker.getInstance(new Configuration());
// assert that the tracker state was cleared.
@@ -117,8 +116,7 @@ public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManaged
String tableName = "testUpdateStatsPreventsAnotherUpdateStatsFromRunning".toUpperCase();
HRegionInfo regionInfo = createTableAndGetRegion(tableName);
markRunningUpdateStats(regionInfo);
- Assert.assertEquals("Row count didn't match", CONCURRENT_UPDATE_STATS_ROW_COUNT,
- runUpdateStats(tableName));
+ assertTrue("Update stats should have been skipped", runUpdateStats(tableName) >= UPDATE_STATS_SKIPPED);
// assert that running the concurrent and race-losing update stats didn't clear the region
// from the tracker. If the method returned true it means the tracker was still tracking
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index aba9c11..fe95470 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -108,7 +108,8 @@ public class IndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String fullTableName = SchemaUtil.getTableName(schemaName, dataTable);
final String indxTable = String.format("%s_%s", dataTable, "INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.TRANSACTIONS_ENABLED, "true");
+ props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
index 0791479..8125007 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
@@ -61,6 +61,7 @@ public class MutableIndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String dataTable = "DATA_TABLE5";
final String indxTable = String.format("%s_%s",dataTable,"INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
index a92a665..c3cdbc0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
@@ -17,24 +17,27 @@
*/
package org.apache.phoenix.end2end;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.Properties;
-import com.google.common.collect.Maps;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertFalse;
+import com.google.common.collect.Maps;
/**
* Verifies that statistics are not collected if they are disabled via a setting
@@ -54,15 +57,21 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
public void testStatisticsAreNotWritten() throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
+ assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
Statement stmt = conn.createStatement();
- stmt.execute("CREATE TABLE T1 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR)");
+ stmt.execute("CREATE TABLE T1 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR) SALT_BUCKETS=3");
stmt.execute("UPSERT INTO T1 VALUES (1, 'NAME1')");
stmt.execute("UPSERT INTO T1 VALUES (2, 'NAME2')");
stmt.execute("UPSERT INTO T1 VALUES (3, 'NAME3')");
conn.commit();
stmt.execute("UPDATE STATISTICS T1");
+ assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
ResultSet rs = stmt.executeQuery("SELECT * FROM SYSTEM.STATS");
assertFalse(rs.next());
+ rs = conn.createStatement().executeQuery("SELECT count(*) FROM T1");
+ assertTrue(rs.next());
+ assertEquals(3,rs.getInt(1));
+ assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
rs.close();
stmt.close();
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index e72f41f..6cd8e13 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -94,6 +94,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
// props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
conn = DriverManager.getConnection(getUrl(), props);
+ assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
conn.createStatement().execute(
"CREATE TABLE " + fullTableName +" ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC))"
@@ -104,6 +105,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// CAll the update statistics query here. If already major compaction has run this will not get executed.
stmt = conn.prepareStatement("UPDATE STATISTICS " + tableName);
stmt.execute();
+ assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
stmt = upsertStmt(conn, tableName);
stmt.setString(1, "z");
s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
@@ -120,6 +122,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
stmt.execute();
rs = conn.createStatement().executeQuery("SELECT k FROM " + tableName);
assertTrue(rs.next());
+ assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
conn.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index ba7eb39..411ce03 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -138,7 +138,6 @@ import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -2717,16 +2716,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
RpcCallback<GetVersionResponse> done) {
GetVersionResponse.Builder builder = GetVersionResponse.newBuilder();
- // The first 3 bytes of the long is used to encoding the HBase version as major.minor.patch.
- // The next 4 bytes of the value is used to encode the Phoenix version as major.minor.patch.
- long version = MetaDataUtil.encodeHBaseAndPhoenixVersions(this.env.getHBaseVersion());
-
- // The last byte is used to communicate whether or not mutable secondary indexing
- // was configured properly.
- version =
- MetaDataUtil.encodeHasIndexWALCodec(version,
- IndexManagementUtil.isWALEditCodecSet(this.env.getConfiguration()));
-
+ long version = MetaDataUtil.encodeVersion(env.getHBaseVersion(), env.getConfiguration());
builder.setVersion(version);
done.run(builder.build());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 6550653..7c98be0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -22,8 +22,9 @@ import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_DISABLED;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_RUN;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_SKIPPED;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -59,7 +60,6 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.exception.DataExceedsCapacityException;
@@ -90,6 +90,7 @@ import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.stats.StatisticsCollector;
import org.apache.phoenix.schema.stats.StatisticsCollectorFactory;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
import org.apache.phoenix.schema.types.PBinary;
import org.apache.phoenix.schema.types.PChar;
@@ -608,7 +609,6 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
InternalScanner internalScanner = scanner;
if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
try {
- Pair<HRegionInfo, HRegionInfo> mergeRegions = null;
long clientTimeStamp = TimeKeeper.SYSTEM.getCurrentTime();
StatisticsCollector stats = StatisticsCollectorFactory.createStatisticsCollector(
c.getEnvironment(), table.getNameAsString(), clientTimeStamp,
@@ -643,22 +643,23 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
if (asyncBytes != null) {
async = Bytes.toBoolean(asyncBytes);
}
- long rowCount = 0; // in case of async, we report 0 as number of rows updated
+ long returnValue = UPDATE_STATS_RUN; // in case of async, we report 1 as number of rows updated
+ boolean statsEnabled = StatisticsUtil.isStatsEnabled(config);
StatisticsCollectionRunTracker statsRunTracker =
StatisticsCollectionRunTracker.getInstance(config);
- boolean runUpdateStats = statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
+ boolean runUpdateStats = statsEnabled && statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
if (runUpdateStats) {
if (!async) {
- rowCount = callable.call();
+ returnValue = callable.call();
} else {
statsRunTracker.runTask(callable);
}
} else {
- rowCount = CONCURRENT_UPDATE_STATS_ROW_COUNT;
- logger.info("UPDATE STATISTICS didn't run because another UPDATE STATISTICS command was already running on the region "
+ returnValue = statsEnabled ? UPDATE_STATS_SKIPPED : UPDATE_STATS_DISABLED;
+ logger.info("UPDATE STATISTICS didn't run because " + (statsEnabled ? " another UPDATE STATISTICS command was already running on the region " : " stats are disabled ")
+ region.getRegionInfo().getRegionNameAsString());
}
- byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
+ byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(returnValue));
final KeyValue aggKeyValue =
KeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
@@ -728,22 +729,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
region.startRegionOperation();
boolean hasMore = false;
boolean noErrors = false;
- boolean compactionRunning = areStatsBeingCollectedViaCompaction();
long rowCount = 0;
+ boolean compactionRunning = areStatsBeingCollectedViaCompaction();
try {
if (!compactionRunning) {
synchronized (innerScanner) {
do {
List<Cell> results = new ArrayList<Cell>();
hasMore = innerScanner.nextRaw(results);
- stats.collectStatistics(results);
rowCount++;
+ stats.collectStatistics(results);
compactionRunning = areStatsBeingCollectedViaCompaction();
} while (hasMore && !compactionRunning);
noErrors = true;
}
}
- return compactionRunning ? COMPACTION_UPDATE_STATS_ROW_COUNT : rowCount;
+ return compactionRunning ? UPDATE_STATS_SKIPPED : UPDATE_STATS_RUN;
} catch (IOException e) {
logger.error("IOException in update stats: " + Throwables.getStackTraceAsString(e));
throw e;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index fc3edbe..8d1a365 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -122,6 +122,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
private long estimatedRows;
private long estimatedSize;
+ private boolean areStatsEnabled;
static final Function<HRegionLocation, KeyRange> TO_KEY_RANGE = new Function<HRegionLocation, KeyRange>() {
@Override
@@ -369,6 +370,11 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return scans;
}
+ // TODO: add to ResultIterators and QueryPlan interfaces?
+ public boolean areStatsEnabled() {
+ return this.areStatsEnabled;
+ }
+
private static List<byte[]> toBoundaries(List<HRegionLocation> regionLocations) {
int nBoundaries = regionLocations.size() - 1;
List<byte[]> ranges = Lists.newArrayListWithExpectedSize(nBoundaries);
@@ -524,6 +530,17 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY;
List<Scan> scans = Lists.newArrayListWithExpectedSize(estGuidepostsPerRegion);
ImmutableBytesWritable guidePosts = gps.getGuidePosts();
+ // If we have any guideposts, then we can definitely say that stats are enabled.
+ // If we have no guideposts, though, we cannot assume that stats are disabled,
+ // as the table may just be too small to have them.
+ if (guidePosts.getLength() > 0) {
+ areStatsEnabled = true;
+ // It's possible that the server was bounced and stats have changed
+ // to become enabled without a client bounce.
+ this.context.getConnection().getQueryServices().setStatsEnabled(true);
+ } else {
+ areStatsEnabled = this.context.getConnection().getQueryServices().areStatsEnabled();
+ }
ByteArrayInputStream stream = null;
DataInput input = null;
PrefixByteDecoder decoder = null;
@@ -851,14 +868,18 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
@Override
public void explain(List<String> planSteps) {
- boolean displayChunkCount = context.getConnection().getQueryServices().getProps().getBoolean(
+ ConnectionQueryServices services = context.getConnection().getQueryServices();
+ boolean displayChunkCount = services.getProps().getBoolean(
QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB,
QueryServicesOptions.DEFAULT_EXPLAIN_CHUNK_COUNT);
+ boolean displayRowCount = services.getProps().getBoolean(
+ QueryServices.EXPLAIN_ROW_COUNT_ATTRIB,
+ QueryServicesOptions.DEFAULT_EXPLAIN_ROW_COUNT);
StringBuilder buf = new StringBuilder();
buf.append("CLIENT ");
if (displayChunkCount) {
buf.append(this.splits.size()).append("-CHUNK ");
- if (estimatedRows > 0) {
+ if (displayRowCount && areStatsEnabled) {
buf.append(estimatedRows).append(" ROWS ");
buf.append(estimatedSize).append(" BYTES ");
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index b5f1f85..471562a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -23,8 +23,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import co.cask.tephra.TransactionSystemClient;
-
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -46,6 +44,8 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
+import co.cask.tephra.TransactionSystemClient;
+
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
public static final int INITIAL_META_DATA_TABLE_CAPACITY = 100;
@@ -123,4 +123,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
TransactionSystemClient getTransactionSystemClient();
public long getRenewLeaseThresholdMilliSeconds();
public boolean isRenewingLeasesEnabled();
+
+ public boolean areStatsEnabled();
+ public void setStatsEnabled(boolean statsEnabled);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index d55ab30..b428eb9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -241,6 +241,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private TransactionServiceClient txServiceClient;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
+ private volatile boolean areStatsEnabled;
// writes guarded by "this"
private volatile boolean closed;
@@ -1108,6 +1109,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void checkClientServerCompatibility() throws SQLException {
StringBuilder buf = new StringBuilder("The following servers require an updated " + QueryConstants.DEFAULT_COPROCESS_PATH + " to be put in the classpath of HBase: ");
boolean isIncompatible = false;
+ boolean areStatsEnabled = false;
int minHBaseVersion = Integer.MAX_VALUE;
try {
List<HRegionLocation> locations = this.getAllTableRegions(SYSTEM_CATALOG_NAME_BYTES);
@@ -1141,18 +1143,23 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
});
for (Map.Entry<byte[],Long> result : results.entrySet()) {
// This is the "phoenix.jar" is in-place, but server is out-of-sync with client case.
- if (!isCompatible(result.getValue())) {
+ long version = result.getValue();
+ // Set stats as being enabled if enabled on *any* region server (though it should
+ // really match across all regions servers).
+ areStatsEnabled |= MetaDataUtil.decodeStatsEnabled(version);
+ if (!isCompatible(version)) {
isIncompatible = true;
HRegionLocation name = regionMap.get(result.getKey());
buf.append(name);
buf.append(';');
}
- hasIndexWALCodec &= hasIndexWALCodec(result.getValue());
- if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
- minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
+ hasIndexWALCodec &= hasIndexWALCodec(version);
+ if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(version)) {
+ minHBaseVersion = MetaDataUtil.decodeHBaseVersion(version);
}
}
lowestClusterHBaseVersion = minHBaseVersion;
+ this.areStatsEnabled = areStatsEnabled;
} catch (SQLException e) {
throw e;
} catch (Throwable t) {
@@ -3379,4 +3386,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ @Override
+ public boolean areStatsEnabled() {
+ return areStatsEnabled;
+ }
+
+ @Override
+ public void setStatsEnabled(boolean statsEnabled) {
+ this.areStatsEnabled = statsEnabled;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index b4bbe1f..453d04f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -76,6 +76,7 @@ import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -106,7 +107,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private final Map<SequenceKey, SequenceInfo> sequenceMap = Maps.newHashMap();
private final String userName;
private final TransactionSystemClient txSystemClient;
- private KeyValueBuilder kvBuilder;
+ private final KeyValueBuilder kvBuilder;
+ private volatile boolean areStatsEnabled;
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
@@ -136,6 +138,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
TransactionManager txnManager = new TransactionManager(config);
this.txSystemClient = new InMemoryTxSystemClient(txnManager);
+ // Just check the properties on the client side (instead of normally the server side)
+ this.areStatsEnabled = StatisticsUtil.isStatsEnabled(config);
}
private PMetaData newEmptyMetaData() {
@@ -527,6 +531,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return txSystemClient;
}
+ @Override
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return new MetaDataMutationResult(MutationCode.FUNCTION_NOT_FOUND, 0l, null);
@@ -578,6 +583,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return false;
}
+ @Override
public HRegionLocation getTableRegionLocation(byte[] tableName, byte[] row) throws SQLException {
List<HRegionLocation> regions = tableSplits.get(Bytes.toString(tableName));
if (regions != null) {
@@ -592,4 +598,14 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
new HRegionInfo(TableName.valueOf(tableName), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW),
SERVER_NAME, -1);
}
+
+ @Override
+ public boolean areStatsEnabled() {
+ return areStatsEnabled;
+ }
+
+ @Override
+ public void setStatsEnabled(boolean statsEnabled) {
+ this.areStatsEnabled = statsEnabled;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 4c7446b..2d0f677 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -310,4 +310,14 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
throws SQLException {
return getDelegate().getTableRegionLocation(tableName, row);
}
+
+ @Override
+ public boolean areStatsEnabled() {
+ return getDelegate().areStatsEnabled();
+ }
+
+ @Override
+ public void setStatsEnabled(boolean statsEnabled) {
+ getDelegate().setStatsEnabled(statsEnabled);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 1efcd8c..32b4fc0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -162,6 +162,7 @@ public interface QueryServices extends SQLCloseable {
public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets";
public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority";
public static final String EXPLAIN_CHUNK_COUNT_ATTRIB = "phoenix.explain.displayChunkCount";
+ public static final String EXPLAIN_ROW_COUNT_ATTRIB = "phoenix.explain.displayRowCount";
public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 27c5693..3e0ffe1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -29,6 +29,7 @@ import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB
import static org.apache.phoenix.query.QueryServices.DELAY_FOR_SCHEMA_UPDATE_CHECK;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.EXPLAIN_ROW_COUNT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GLOBAL_METRICS_ENABLED;
@@ -196,6 +197,7 @@ public class QueryServicesOptions {
*/
public static final int DEFAULT_COPROCESSOR_PRIORITY = Coprocessor.PRIORITY_SYSTEM/2 + Coprocessor.PRIORITY_USER/2; // Divide individually to prevent any overflow
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = true;
+ public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = true;
public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
@@ -573,6 +575,11 @@ public class QueryServicesOptions {
return this;
}
+ public QueryServicesOptions setExplainRowCount(boolean showRowCount) {
+ config.setBoolean(EXPLAIN_ROW_COUNT_ATTRIB, showRowCount);
+ return this;
+ }
+
public QueryServicesOptions setAllowOnlineSchemaUpdate(boolean allow) {
config.setBoolean(ALLOW_ONLINE_TABLE_SCHEMA_UPDATE, allow);
return this;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 7f3f850..eee42ea 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -182,6 +182,7 @@ import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
@@ -955,6 +956,7 @@ public class MetaDataClient {
}
MutationState mutationState = plan.execute();
rowCount = mutationState.getUpdateCount();
+ this.getConnection().getQueryServices().setStatsEnabled(rowCount != StatisticsCollectionRunTracker.UPDATE_STATS_DISABLED);
}
/*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index 1063229..6c73f16 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -21,13 +21,10 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
index 4ed3325..6413e37 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
@@ -24,7 +24,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -45,9 +44,9 @@ public class StatisticsCollectionRunTracker {
.newSetFromMap(new ConcurrentHashMap<HRegionInfo, Boolean>());
private final ExecutorService executor;
- // Constants added for testing purposes
- public static final long CONCURRENT_UPDATE_STATS_ROW_COUNT = -100l;
- public static final long COMPACTION_UPDATE_STATS_ROW_COUNT = -200l;
+ public static final long UPDATE_STATS_RUN = 1L;
+ public static final long UPDATE_STATS_SKIPPED = 100000L;
+ public static final long UPDATE_STATS_DISABLED = 0;
public static StatisticsCollectionRunTracker getInstance(Configuration config) {
StatisticsCollectionRunTracker result = INSTANCE;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 5b47104..9dd72e8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
@@ -40,6 +41,8 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
@@ -220,4 +223,17 @@ public class StatisticsUtil {
ptr.set(row, gpOffset, row.length - gpOffset);
return ByteUtil.copyKeyBytesIfNecessary(ptr);
}
+
+ public static boolean isStatsEnabled(Configuration conf) {
+ if (conf.getBoolean(QueryServices.STATS_ENABLED_ATTRIB, true)) {
+ if (conf.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) != 1) {
+ if (conf.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) < HConstants.DEFAULT_MAX_FILE_SIZE) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 6d5903d..5ee3c0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -56,6 +57,7 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PLong;
@@ -103,26 +105,25 @@ public class MetaDataUtil {
// The second byte in int would be the major version, 3rd byte minor version, and 4th byte
// patch version.
public static int decodePhoenixVersion(long version) {
- return (int) ((version << Byte.SIZE * 3) >>> Byte.SIZE * 4);
- }
-
- // TODO: generalize this to use two bytes to return a SQL error code instead
- public static long encodeHasIndexWALCodec(long version, boolean isValid) {
- if (!isValid) {
- return version | 1;
- }
- return version;
+ return (int) ((version << Byte.SIZE * 4) >>> Byte.SIZE * 5);
}
public static boolean decodeHasIndexWALCodec(long version) {
return (version & 0xF) == 0;
}
+ // Given the encoded integer representing the phoenix version in the encoded version value.
+ // The second byte in int would be the major version, 3rd byte minor version, and 4th byte
+ // patch version.
+ public static boolean decodeStatsEnabled(long version) {
+ return ((int) ((version << Byte.SIZE * 3) >>> Byte.SIZE * 7) & 0x1) != 0;
+ }
+
// Given the encoded integer representing the client hbase version in the encoded version value.
// The second byte in int would be the major version, 3rd byte minor version, and 4th byte
// patch version.
public static int decodeHBaseVersion(long version) {
- return (int) (version >>> Byte.SIZE * 5);
+ return (int) (version >>> (Byte.SIZE * 5));
}
public static String decodeHBaseVersionAsString(int version) {
@@ -132,17 +133,33 @@ public class MetaDataUtil {
return major + "." + minor + "." + patch;
}
- public static int encodePhoenixVersion() {
- return VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ // The first 3 bytes of the long is used to encoding the HBase version as major.minor.patch.
+ // The next 4 bytes of the value is used to encode the Phoenix version as major.minor.patch.
+ /**
+ * Encode HBase and Phoenix version along with some server-side config information such
+ * as whether WAL codec is installed (necessary for non transactional, mutable secondar
+ * indexing), and whether stats are enabled.
+ * @param env RegionCoprocessorEnvironment to access HBase version and Configuration.
+ * @return long value sent back during initialization of a cluster connection.
+ */
+ public static long encodeVersion(String hbaseVersionStr, Configuration config) {
+ long hbaseVersion = VersionUtil.encodeVersion(hbaseVersionStr);
+ long statsEnabled = StatisticsUtil.isStatsEnabled(config) ? 1 : 0;
+ long phoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
MetaDataProtocol.PHOENIX_PATCH_NUMBER);
+ long walCodec = IndexManagementUtil.isWALEditCodecSet(config) ? 0 : 1;
+ long version =
+ // Encode HBase major, minor, patch version
+ (hbaseVersion << (Byte.SIZE * 5))
+ // Encode if stats are enabled on the server side
+ | (statsEnabled << (Byte.SIZE * 4))
+ // Encode Phoenix major, minor, patch version
+ | (phoenixVersion << (Byte.SIZE * 1))
+ // Encode whether or not non transactional, mutable secondary indexing was configured properly.
+ | walCodec;
+ return version;
}
-
- public static long encodeHBaseAndPhoenixVersions(String hbaseVersion) {
- return (((long) VersionUtil.encodeVersion(hbaseVersion)) << (Byte.SIZE * 5)) |
- (((long) VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
- MetaDataProtocol.PHOENIX_PATCH_NUMBER)) << (Byte.SIZE * 1));
- }
-
+
public static void getTenantIdAndSchemaAndTableName(List<Mutation> tableMetadata, byte[][] rowKeyMetaData) {
Mutation m = getTableHeaderRow(tableMetadata);
getVarChars(m.getRow(), 3, rowKeyMetaData);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 29a7001..6ae655c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -54,6 +54,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = false; // TODO: update explain plans in test and set to true
+ public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = false; // TODO: update explain plans in test and set to true
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
@@ -80,6 +81,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static QueryServicesOptions getDefaultServicesOptions() {
return withDefaults()
.setExplainChunkCount(DEFAULT_EXPLAIN_CHUNK_COUNT)
+ .setExplainRowCount(DEFAULT_EXPLAIN_ROW_COUNT)
.setSequenceSaltBuckets(DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS)
.setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a1e6ae44/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
index 1e06379..7ffc054 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
@@ -21,15 +21,19 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.HBaseFactoryProvider;
+import org.apache.phoenix.query.QueryServices;
import org.junit.Test;
@@ -58,6 +62,35 @@ public class MetaDataUtilTest {
assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 3, 5));
}
+ @Test
+ public void testEncodeDecode() {
+ String hbaseVersionStr = "0.98.14";
+ Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
+ config.setBoolean(QueryServices.STATS_ENABLED_ATTRIB, false);
+
+ long version = MetaDataUtil.encodeVersion(hbaseVersionStr, config);
+ int hbaseVersion = MetaDataUtil.decodeHBaseVersion(version);
+ int expectedHBaseVersion = VersionUtil.encodeVersion(0, 98, 14);
+ assertEquals(expectedHBaseVersion, hbaseVersion);
+ boolean areStatsEnabled = MetaDataUtil.decodeStatsEnabled(version);
+ assertFalse(areStatsEnabled);
+ int phoenixVersion = MetaDataUtil.decodePhoenixVersion(version);
+ int expectedPhoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ MetaDataProtocol.PHOENIX_PATCH_NUMBER);
+ assertEquals(expectedPhoenixVersion, phoenixVersion);
+
+ config.setBoolean(QueryServices.STATS_ENABLED_ATTRIB, true);
+ version = MetaDataUtil.encodeVersion(hbaseVersionStr, config);
+ hbaseVersion = MetaDataUtil.decodeHBaseVersion(version);
+ expectedHBaseVersion = VersionUtil.encodeVersion(0, 98, 14);
+ assertEquals(expectedHBaseVersion, hbaseVersion);
+ areStatsEnabled = MetaDataUtil.decodeStatsEnabled(version);
+ assertTrue(areStatsEnabled);
+ phoenixVersion = MetaDataUtil.decodePhoenixVersion(version);
+ expectedPhoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ MetaDataProtocol.PHOENIX_PATCH_NUMBER);
+ assertEquals(expectedPhoenixVersion, phoenixVersion);
+ }
/**
* Ensure it supports {@link GenericKeyValueBuilder}
* @throws Exception on failure
[12/12] phoenix git commit: Temporary fix for PHOENIX-2714
Posted by ma...@apache.org.
Temporary fix for PHOENIX-2714
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/36ee23ab
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/36ee23ab
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/36ee23ab
Branch: refs/heads/calcite
Commit: 36ee23ab7760616394f32f88b2da6b038cfee1e9
Parents: 06fd581
Author: maryannxue <ma...@gmail.com>
Authored: Thu Feb 25 15:38:55 2016 -0500
Committer: maryannxue <ma...@gmail.com>
Committed: Thu Feb 25 15:38:55 2016 -0500
----------------------------------------------------------------------
.../phoenix/calcite/rel/PhoenixTableScan.java | 4 +-
.../apache/phoenix/execute/AggregatePlan.java | 2 -
.../apache/phoenix/execute/BaseQueryPlan.java | 11 --
.../org/apache/phoenix/execute/ScanPlan.java | 2 -
.../phoenix/iterate/BaseResultIterators.java | 154 ++++++++++++++++---
5 files changed, 137 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/36ee23ab/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
index 80c328f..dfdb507 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
@@ -36,6 +36,7 @@ import org.apache.phoenix.execute.ScanPlan;
import org.apache.phoenix.execute.TupleProjector;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.LiteralExpression;
+import org.apache.phoenix.iterate.BaseResultIterators;
import org.apache.phoenix.iterate.ParallelIteratorFactory;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.parse.SelectStatement;
@@ -131,8 +132,7 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel {
filterExpr = WhereOptimizer.pushKeyExpressionsToScan(context, select, filterExpr);
WhereCompiler.setScanFilter(context, select, filterExpr, true, false);
scanRanges = context.getScanRanges();
- ScanPlan plan = new ScanPlan(context, select, tableRef, RowProjector.EMPTY_PROJECTOR, null, OrderBy.EMPTY_ORDER_BY, null, true, null);
- estimatedSize = plan.getEstimatedBytes();
+ estimatedSize = BaseResultIterators.getEstimatedCount(context, tableRef.getTable()).getSecond();
} catch (SQLException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/36ee23ab/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
index f9eca0c..794247b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
@@ -208,8 +208,6 @@ public class AggregatePlan extends BaseQueryPlan {
splits = iterators.getSplits();
scans = iterators.getScans();
- estimatedSize = iterators.getEstimatedByteCount();
- estimatedRows = iterators.getEstimatedRowCount();
AggregatingResultIterator aggResultIterator;
// No need to merge sort for ungrouped aggregation
http://git-wip-us.apache.org/repos/asf/phoenix/blob/36ee23ab/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 0ee70ba..f389fd0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -108,8 +108,6 @@ public abstract class BaseQueryPlan implements QueryPlan {
* immediately before creating the ResultIterator.
*/
protected final Expression dynamicFilter;
- protected Long estimatedRows;
- protected Long estimatedSize;
protected BaseQueryPlan(
@@ -129,16 +127,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
this.parallelIteratorFactory = parallelIteratorFactory;
this.dynamicFilter = dynamicFilter;
}
-
- public Long getEstimatedRowCount() {
- return this.estimatedRows;
- }
- public Long getEstimatedByteCount() {
- return this.estimatedSize;
- }
-
-
@Override
public Operation getOperation() {
return Operation.QUERY;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/36ee23ab/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index 72d95be..23bf435 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -208,8 +208,6 @@ public class ScanPlan extends BaseQueryPlan {
}
splits = iterators.getSplits();
scans = iterators.getScans();
- estimatedSize = iterators.getEstimatedByteCount();
- estimatedRows = iterators.getEstimatedRowCount();
if (isOrdered) {
scanner = new MergeSortTopNResultIterator(iterators, limit, orderBy.getOrderByExpressions());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/36ee23ab/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index a48de13..22c0197 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -113,7 +113,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final List<List<Scan>> scans;
private final List<KeyRange> splits;
- private final PTableStats tableStats;
private final byte[] physicalTableName;
private final QueryPlan plan;
protected final String scanId;
@@ -136,7 +135,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return plan.getTableRef().getTable();
}
- private boolean useStats() {
+ private static boolean useStats(StatementContext context) {
Scan scan = context.getScan();
boolean isPointLookup = context.getScanRanges().isPointLookup();
/*
@@ -337,7 +336,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
TableRef tableRef = plan.getTableRef();
PTable table = tableRef.getTable();
physicalTableName = table.getPhysicalName().getBytes();
- tableStats = useStats() ? new MetaDataClient(context.getConnection()).getTableStats(table) : PTableStats.EMPTY_STATS;
// Used to tie all the scans together during logging
scanId = UUID.randomUUID().toString();
@@ -397,18 +395,18 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return guideIndex;
}
- private GuidePostsInfo getGuidePosts(Set<byte[]> whereConditions) {
+ private static GuidePostsInfo getGuidePosts(StatementContext context, PTable table, Set<byte[]> whereConditions) throws SQLException {
/*
* Don't use guide posts if: 1) We're doing a point lookup, as HBase is fast enough at those to not need them to
* be further parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan
* entire regions worth of data to track where to put the guide posts.
*/
- if (!useStats()) { return GuidePostsInfo.NO_GUIDEPOST; }
+ if (!useStats(context)) { return GuidePostsInfo.NO_GUIDEPOST; }
GuidePostsInfo gps = null;
- PTable table = getTable();
+ PTableStats tableStats = new MetaDataClient(context.getConnection()).getTableStats(table);
Map<byte[], GuidePostsInfo> guidePostMap = tableStats.getGuidePosts();
- byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable());
+ byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(table);
if (table.getColumnFamilies().isEmpty()) {
// For sure we can get the defaultCF from the table
gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
@@ -430,7 +428,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return gps;
}
- private GuidePostsInfo getDefaultFamilyGuidePosts(Map<byte[], GuidePostsInfo> guidePostMap, byte[] defaultCF) {
+ private static GuidePostsInfo getDefaultFamilyGuidePosts(Map<byte[], GuidePostsInfo> guidePostMap, byte[] defaultCF) {
if (guidePostMap.get(defaultCF) != null) {
return guidePostMap.get(defaultCF);
}
@@ -491,7 +489,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
whereConditions.add(cf);
}
}
- GuidePostsInfo gps = getGuidePosts(whereConditions);
+ GuidePostsInfo gps = getGuidePosts(context, table, whereConditions);
hasGuidePosts = gps != GuidePostsInfo.NO_GUIDEPOST;
boolean traverseAllRegions = isSalted || isLocalIndex;
if (!traverseAllRegions) {
@@ -566,8 +564,10 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
false);
- estimatedRows += gps.getRowCounts().get(guideIndex);
- estimatedSize += gps.getByteCounts().get(guideIndex);
+ if (newScan != null) {
+ estimatedRows += gps.getRowCounts().get(guideIndex);
+ estimatedSize += gps.getByteCounts().get(guideIndex);
+ }
scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
currentKeyBytes = currentGuidePost.copyBytes();
currentGuidePost = PrefixByteCodec.decode(decoder, input);
@@ -603,6 +603,130 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return parallelScans;
}
+
+ /**
+ * Compute the estimated count of rows and bytes that will be scanned.
+ * @return the estimated row count and the byte count.
+ * @throws SQLException
+ */
+ public static Pair<Long, Long> getEstimatedCount(StatementContext context, PTable table) throws SQLException {
+ if (table.getName() == null) { // empty table
+ return new Pair<Long, Long>(null, null);
+ }
+
+ if (context.getScanRanges().isPointLookup()) {
+ return new Pair<Long, Long>(1L, SchemaUtil.estimateRowSize(table));
+ }
+
+ TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
+ byte[] cf = where.getFirst();
+ if (cf != null) {
+ whereConditions.add(cf);
+ }
+ }
+ GuidePostsInfo gps = getGuidePosts(context, table, whereConditions);
+ if (gps == GuidePostsInfo.NO_GUIDEPOST) {
+ return new Pair<Long, Long>(null, null);
+ }
+
+ byte[] startKey = ByteUtil.EMPTY_BYTE_ARRAY;
+ byte[] stopKey = ByteUtil.EMPTY_BYTE_ARRAY;
+ Scan scan = context.getScan();
+ List<HRegionLocation> regionLocations = context.getConnection().getQueryServices()
+ .getAllTableRegions(table.getPhysicalName().getBytes());
+ List<byte[]> regionBoundaries = toBoundaries(regionLocations);
+ ScanRanges scanRanges = context.getScanRanges();
+ boolean isSalted = table.getBucketNum() != null;
+ boolean isLocalIndex = table.getIndexType() == IndexType.LOCAL;
+ boolean traverseAllRegions = isSalted || isLocalIndex;
+ if (!traverseAllRegions) {
+ byte[] scanStartRow = scan.getStartRow();
+ if (scanStartRow.length != 0 && Bytes.compareTo(scanStartRow, startKey) > 0) {
+ startKey = scanStartRow;
+ }
+ byte[] scanStopRow = scan.getStopRow();
+ if (stopKey.length == 0
+ || (scanStopRow.length != 0 && Bytes.compareTo(scanStopRow, stopKey) < 0)) {
+ stopKey = scanStopRow;
+ }
+ }
+
+ int regionIndex = 0;
+ int stopIndex = regionBoundaries.size();
+ if (startKey.length > 0) {
+ regionIndex = getIndexContainingInclusive(regionBoundaries, startKey);
+ }
+ if (stopKey.length > 0) {
+ stopIndex = Math.min(stopIndex, regionIndex + getIndexContainingExclusive(regionBoundaries.subList(regionIndex, stopIndex), stopKey));
+ if (isLocalIndex) {
+ stopKey = regionLocations.get(stopIndex).getRegionInfo().getEndKey();
+ }
+ }
+
+ ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey);
+
+ int gpsSize = gps.getGuidePostsCount();
+ int keyOffset = 0;
+ ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY;
+ ImmutableBytesWritable guidePosts = gps.getGuidePosts();
+ ByteArrayInputStream stream = null;
+ DataInput input = null;
+ PrefixByteDecoder decoder = null;
+ int guideIndex = 0;
+ long estimatedRows = 0;
+ long estimatedSize = 0;
+ try {
+ if (gpsSize > 0) {
+ stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength());
+ input = new DataInputStream(stream);
+ decoder = new PrefixByteDecoder(gps.getMaxLength());
+ try {
+ while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0
+ && currentKey.getLength() != 0) {
+ guideIndex++;
+ }
+ } catch (EOFException e) {}
+ }
+ byte[] currentKeyBytes = currentKey.copyBytes();
+
+ // Merge bisect with guideposts for all but the last region
+ while (regionIndex <= stopIndex) {
+ byte[] currentGuidePostBytes = currentGuidePost.copyBytes();
+ byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY;
+ if (regionIndex == stopIndex) {
+ endKey = stopKey;
+ } else {
+ endKey = regionBoundaries.get(regionIndex);
+ }
+ HRegionLocation regionLocation = regionLocations.get(regionIndex);
+ if (isLocalIndex) {
+ HRegionInfo regionInfo = regionLocation.getRegionInfo();
+ endRegionKey = regionInfo.getEndKey();
+ keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey);
+ }
+ try {
+ while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
+ Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
+ false);
+ if (newScan != null) {
+ estimatedRows += gps.getRowCounts().get(guideIndex);
+ estimatedSize += gps.getByteCounts().get(guideIndex);
+ }
+ currentKeyBytes = currentGuidePost.copyBytes();
+ currentGuidePost = PrefixByteCodec.decode(decoder, input);
+ currentGuidePostBytes = currentGuidePost.copyBytes();
+ guideIndex++;
+ }
+ } catch (EOFException e) {}
+ currentKeyBytes = endKey;
+ regionIndex++;
+ }
+ } finally {
+ if (stream != null) Closeables.closeQuietly(stream);
+ }
+ return new Pair<Long, Long>(estimatedRows, estimatedSize);
+ }
public static <T> List<T> reverseIfNecessary(List<T> list, boolean reverse) {
if (!reverse) {
@@ -884,14 +1008,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
buf.append(getName()).append(" ").append(size()).append("-WAY ");
explain(buf.toString(),planSteps);
}
-
- public Long getEstimatedRowCount() {
- return this.estimatedRows;
- }
-
- public Long getEstimatedByteCount() {
- return this.estimatedSize;
- }
@Override
public String toString() {
[11/12] phoenix git commit: Merge remote-tracking branch
'origin/master' into calcite
Posted by ma...@apache.org.
Merge remote-tracking branch 'origin/master' into calcite
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/06fd5812
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/06fd5812
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/06fd5812
Branch: refs/heads/calcite
Commit: 06fd581296bc58390ebf7f8e738af19a6634a66e
Parents: 87e8f29 29ca38b
Author: maryannxue <ma...@gmail.com>
Authored: Thu Feb 25 15:23:38 2016 -0500
Committer: maryannxue <ma...@gmail.com>
Committed: Thu Feb 25 15:23:38 2016 -0500
----------------------------------------------------------------------
phoenix-assembly/src/build/server.xml | 1 +
.../org/apache/phoenix/end2end/IndexToolIT.java | 3 +-
.../phoenix/end2end/MutableIndexToolIT.java | 1 +
.../end2end/StatsCollectionDisabledIT.java | 21 ++-
.../phoenix/end2end/StatsCollectorIT.java | 57 +++++++-
.../phoenix/coprocessor/MetaDataProtocol.java | 2 +-
.../UngroupedAggregateRegionObserver.java | 4 +-
.../apache/phoenix/execute/AggregatePlan.java | 6 +-
.../apache/phoenix/execute/BaseQueryPlan.java | 11 ++
.../org/apache/phoenix/execute/ScanPlan.java | 7 +-
.../phoenix/iterate/BaseResultIterators.java | 140 +++++++++++--------
.../query/ConnectionQueryServicesImpl.java | 86 ++++++++++--
.../apache/phoenix/query/QueryConstants.java | 2 +-
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 7 +
.../stats/DefaultStatisticsCollector.java | 33 ++---
.../phoenix/schema/stats/GuidePostsInfo.java | 2 +-
.../schema/stats/GuidePostsInfoBuilder.java | 6 -
.../phoenix/schema/stats/PTableStatsImpl.java | 5 +-
.../stats/StatisticsCollectorFactory.java | 17 ++-
.../phoenix/schema/stats/StatisticsUtil.java | 23 +--
.../phoenix/schema/stats/StatisticsWriter.java | 60 ++++----
.../apache/phoenix/util/PrefixByteCodec.java | 37 ++---
.../phoenix/query/QueryServicesTestImpl.java | 2 +
24 files changed, 360 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/06fd5812/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/06fd5812/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/06fd5812/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/06fd5812/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
[08/12] phoenix git commit: PHOENIX-2713 Backward compatibility fails
with CNF exception with 4.7 client and server when upgraded from 4.6
Posted by ma...@apache.org.
PHOENIX-2713 Backward compatibility fails with CNF exception with 4.7 client and server when upgraded from 4.6
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a6de19bd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a6de19bd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a6de19bd
Branch: refs/heads/calcite
Commit: a6de19bd2290d8242b002ff661ec7028577e3055
Parents: 1f7b47a
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Feb 24 17:25:51 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Feb 24 17:28:26 2016 -0800
----------------------------------------------------------------------
.../query/ConnectionQueryServicesImpl.java | 65 ++++++++++++++++----
.../stats/StatisticsCollectorFactory.java | 17 ++++-
2 files changed, 68 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a6de19bd/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index d27a4bc..37ebc78 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2367,24 +2367,24 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
}
if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
+ // Drop old stats table so that new stats table is created
+ metaConnection = dropStatsTable(metaConnection,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
// Add these columns one at a time, each with different timestamps so that if folks have
// run the upgrade code already for a snapshot, we'll still enter this block (and do the
// parts we haven't yet done).
- metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4,
+ metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName());
- // Drop old stats table so that new stats table is created
- metaConnection = dropStatsTable(metaConnection,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3);
- metaConnection = addColumnsIfNotExists(metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
- PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
- + PLong.INSTANCE.getSqlTypeName());
- metaConnection = setImmutableTableIndexesImmutable(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
- Properties props = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0));
- metaConnection = new PhoenixConnection(metaConnection, ConnectionQueryServicesImpl.this, props);
- // that already have cached data.
+ PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " " + PLong.INSTANCE.getSqlTypeName());
+ metaConnection = setImmutableTableIndexesImmutable(metaConnection,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
+ metaConnection = updateSystemCatalogTimestamp(metaConnection,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+ ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+ logger.warn("Update of SYSTEM.CATALOG complete");
clearCache();
}
@@ -2535,6 +2535,45 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return metaConnection;
}
+ /**
+ * Forces update of SYSTEM.CATALOG by setting column to existing value
+ * @param oldMetaConnection
+ * @param timestamp
+ * @return
+ * @throws SQLException
+ */
+ private PhoenixConnection updateSystemCatalogTimestamp(PhoenixConnection oldMetaConnection, long timestamp) throws SQLException {
+ SQLException sqlE = null;
+ Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
+ PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props);
+ boolean autoCommit = metaConnection.getAutoCommit();
+ try {
+ metaConnection.setAutoCommit(true);
+ metaConnection.createStatement().execute(
+ "UPSERT INTO SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, DISABLE_WAL)\n" +
+ "VALUES (NULL, '" + QueryConstants.SYSTEM_SCHEMA_NAME + "','" + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE + "', NULL, NULL, FALSE)");
+ } catch (SQLException e) {
+ logger.warn("exception during upgrading stats table:" + e);
+ sqlE = e;
+ } finally {
+ try {
+ metaConnection.setAutoCommit(autoCommit);
+ oldMetaConnection.close();
+ } catch (SQLException e) {
+ if (sqlE != null) {
+ sqlE.setNextException(e);
+ } else {
+ sqlE = e;
+ }
+ }
+ if (sqlE != null) {
+ throw sqlE;
+ }
+ }
+ return metaConnection;
+ }
+
private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, long timestamp)
throws SQLException, IOException {
Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a6de19bd/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
index aaffd73..964bb28 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
@@ -18,10 +18,15 @@
package org.apache.phoenix.schema.stats;
import java.io.IOException;
+import java.util.Set;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
+import com.google.common.collect.Sets;
+
/**
* Provides new {@link StatisticsCollector} instances based on configuration settings for a
* table (or system-wide configuration of statistics).
@@ -50,6 +55,15 @@ public class StatisticsCollectorFactory {
}
}
+ // TODO: make this declarative through new DISABLE_STATS column on SYSTEM.CATALOG table.
+ // Also useful would be a USE_CURRENT_TIME_FOR_STATS column on SYSTEM.CATALOG table.
+ private static final Set<TableName> DISABLE_STATS = Sets.newHashSetWithExpectedSize(3);
+ static {
+ DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME));
+ DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME));
+ DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SEQUENCE_FULLNAME));
+ }
+
/**
* Determines if statistics are enabled (which is the default). This is done on the
* RegionCoprocessorEnvironment for now to allow setting this on a per-table basis, although
@@ -57,7 +71,8 @@ public class StatisticsCollectorFactory {
* use case for that.
*/
private static boolean statisticsEnabled(RegionCoprocessorEnvironment env) {
- return env.getConfiguration().getBoolean(QueryServices.STATS_ENABLED_ATTRIB, true);
+ return env.getConfiguration().getBoolean(QueryServices.STATS_ENABLED_ATTRIB, true) &&
+ !DISABLE_STATS.contains(env.getRegionInfo().getTable());
}
}
[02/12] phoenix git commit: LP-2705 Downgrade warn logging in region
observer
Posted by ma...@apache.org.
LP-2705 Downgrade warn logging in region observer
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/da51f46b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/da51f46b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/da51f46b
Branch: refs/heads/calcite
Commit: da51f46b2394108f47368e40b5c3abd97dc33714
Parents: a1e6ae4
Author: Gabriel Reid <ga...@ngdata.com>
Authored: Tue Feb 23 08:42:34 2016 +0100
Committer: Gabriel Reid <ga...@ngdata.com>
Committed: Tue Feb 23 08:43:02 2016 +0100
----------------------------------------------------------------------
.../phoenix/coprocessor/UngroupedAggregateRegionObserver.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/da51f46b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 7c98be0..91ec20e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -152,7 +152,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
Mutation[] mutationArray = new Mutation[mutations.size()];
// TODO: should we use the one that is all or none?
- logger.warn("Committing bactch of " + mutations.size() + " mutations for " + region.getRegionInfo().getTable().getNameAsString());
+ logger.debug("Committing bactch of " + mutations.size() + " mutations for " + region.getRegionInfo().getTable().getNameAsString());
region.batchMutate(mutations.toArray(mutationArray), HConstants.NO_NONCE, HConstants.NO_NONCE);
}
@@ -209,7 +209,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
byte[] descRowKeyTableBytes = scan.getAttribute(UPGRADE_DESC_ROW_KEY);
boolean isDescRowKeyOrderUpgrade = descRowKeyTableBytes != null;
if (isDescRowKeyOrderUpgrade) {
- logger.warn("Upgrading row key for " + region.getRegionInfo().getTable().getNameAsString());
+ logger.debug("Upgrading row key for " + region.getRegionInfo().getTable().getNameAsString());
projectedTable = deserializeTable(descRowKeyTableBytes);
try {
writeToTable = PTableImpl.makePTable(projectedTable, true);
[06/12] phoenix git commit: PHOENIX-2711 Need Thrift included in
phoenix-server jar for M/R index (Thomas D'Silva)
Posted by ma...@apache.org.
PHOENIX-2711 Need Thrift included in phoenix-server jar for M/R index (Thomas D'Silva)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/05e6e2a3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/05e6e2a3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/05e6e2a3
Branch: refs/heads/calcite
Commit: 05e6e2a3d67ed5c7c1cace0b2903c35b83ee5ec5
Parents: e3a6202
Author: James Taylor <ja...@apache.org>
Authored: Tue Feb 23 17:13:10 2016 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Tue Feb 23 17:16:58 2016 -0800
----------------------------------------------------------------------
phoenix-assembly/src/build/server.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/05e6e2a3/phoenix-assembly/src/build/server.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/server.xml b/phoenix-assembly/src/build/server.xml
index ef79486..e8f9bfb 100644
--- a/phoenix-assembly/src/build/server.xml
+++ b/phoenix-assembly/src/build/server.xml
@@ -46,6 +46,7 @@
<include>com.google.inject.extensions:guice-assistedinject</include>
<include>it.unimi.dsi:fastutil</include>
<include>io.dropwizard.metrics:metrics-core</include>
+ <include>org.apache.thrift:libthrift</include>
</includes>
</dependencySet>
</dependencySets>
[10/12] phoenix git commit: Revert changes for PHOENIX-2714
Posted by ma...@apache.org.
Revert changes for PHOENIX-2714
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/87e8f29b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/87e8f29b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/87e8f29b
Branch: refs/heads/calcite
Commit: 87e8f29bb2f93b875c73dce499cf5b95044072c4
Parents: 9ac854a
Author: maryannxue <ma...@gmail.com>
Authored: Thu Feb 25 15:23:03 2016 -0500
Committer: maryannxue <ma...@gmail.com>
Committed: Thu Feb 25 15:23:03 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/phoenix/execute/ScanPlan.java | 5 -----
.../apache/phoenix/iterate/BaseResultIterators.java | 10 ++--------
.../org/apache/phoenix/iterate/ResultIterators.java | 1 -
.../apache/phoenix/iterate/UnionResultIterators.java | 5 -----
.../phoenix/iterate/AggregateResultScannerTest.java | 5 -----
.../phoenix/iterate/ConcatResultIteratorTest.java | 10 ----------
.../phoenix/iterate/MergeSortResultIteratorTest.java | 15 ---------------
7 files changed, 2 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index f622e8e..673a641 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -34,7 +34,6 @@ import org.apache.phoenix.coprocessor.ScanRegionObserver;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.iterate.ChunkedResultIterator;
import org.apache.phoenix.iterate.ConcatResultIterator;
-import org.apache.phoenix.iterate.DefaultParallelScanGrouper;
import org.apache.phoenix.iterate.LimitingResultIterator;
import org.apache.phoenix.iterate.MergeSortRowKeyResultIterator;
import org.apache.phoenix.iterate.MergeSortTopNResultIterator;
@@ -258,8 +257,4 @@ public class ScanPlan extends BaseQueryPlan {
}
}
- public long getEstimatedBytes() throws SQLException {
- return new ParallelIterators(this, null, parallelIteratorFactory, DefaultParallelScanGrouper.getInstance()).getEstimatedSize();
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index c3da407..fc3edbe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -560,10 +560,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
false);
- if (newScan != null) {
- estimatedRows += gps.getRowCounts().get(guideIndex);
- estimatedSize += gps.getByteCounts().get(guideIndex);
- }
+ estimatedRows += gps.getRowCounts().get(guideIndex);
+ estimatedSize += gps.getByteCounts().get(guideIndex);
scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
currentKeyBytes = currentGuidePost.copyBytes();
currentGuidePost = PrefixByteCodec.decode(decoder, input);
@@ -874,8 +872,4 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]";
}
- @Override
- public long getEstimatedSize() {
- return estimatedSize;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ResultIterators.java
index e76db56..16f8b41 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ResultIterators.java
@@ -26,7 +26,6 @@ import org.apache.phoenix.util.SQLCloseable;
public interface ResultIterators extends SQLCloseable {
public int size();
- public long getEstimatedSize();
public List<KeyRange> getSplits();
public List<List<Scan>> getScans();
public void explain(List<String> planSteps);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/main/java/org/apache/phoenix/iterate/UnionResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/UnionResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/UnionResultIterators.java
index 0c2cfea..5018dad 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/UnionResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/UnionResultIterators.java
@@ -139,9 +139,4 @@ public class UnionResultIterators implements ResultIterators {
public List<PeekingResultIterator> getIterators() throws SQLException {
return iterators;
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java
index 60e4b2d..f53e871 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java
@@ -168,11 +168,6 @@ public class AggregateResultScannerTest extends BaseConnectionlessQueryTest {
@Override
public void close() throws SQLException {
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
};
ResultIterator scanner = new GroupedAggregatingResultIterator(new MergeSortRowKeyResultIterator(iterators), aggregationManager.getAggregators());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/test/java/org/apache/phoenix/iterate/ConcatResultIteratorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/ConcatResultIteratorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/ConcatResultIteratorTest.java
index 2670c6d..cf71724 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/ConcatResultIteratorTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/ConcatResultIteratorTest.java
@@ -85,11 +85,6 @@ public class ConcatResultIteratorTest {
@Override
public void close() throws SQLException {
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
};
Tuple[] expectedResults = new Tuple[] {
@@ -153,11 +148,6 @@ public class ConcatResultIteratorTest {
@Override
public void close() throws SQLException {
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
};
ResultIterator scanner = new MergeSortRowKeyResultIterator(iterators);
AssertResults.assertResults(scanner, expectedResults);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87e8f29b/phoenix-core/src/test/java/org/apache/phoenix/iterate/MergeSortResultIteratorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/MergeSortResultIteratorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/MergeSortResultIteratorTest.java
index 6e3e0b7..9b2e8de 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/MergeSortResultIteratorTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/MergeSortResultIteratorTest.java
@@ -100,11 +100,6 @@ public class MergeSortResultIteratorTest {
@Override
public void close() throws SQLException {
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
};
ResultIterators reverseIterators = new ResultIterators() {
@@ -135,11 +130,6 @@ public class MergeSortResultIteratorTest {
@Override
public void close() throws SQLException {
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
};
ResultIterator scanner = new MergeSortRowKeyResultIterator(iterators);
AssertResults.assertResults(scanner, expectedResults);
@@ -203,11 +193,6 @@ public class MergeSortResultIteratorTest {
@Override
public void close() throws SQLException {
}
-
- @Override
- public long getEstimatedSize() {
- return 0;
- }
};
ResultIterator scanner = new MergeSortRowKeyResultIterator(iterators, 0, true);
AssertResults.assertResults(scanner, expectedResults);
[05/12] phoenix git commit: PHOENIX-2707 Differentiate between a
table+family have zero guideposts from not having collected guideposts
Posted by ma...@apache.org.
PHOENIX-2707 Differentiate between a table+family have zero guideposts from not having collected guideposts
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e3a62026
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e3a62026
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e3a62026
Branch: refs/heads/calcite
Commit: e3a62026cbfaf1eb7c50a35a8871e0f059d7d668
Parents: c25f887
Author: James Taylor <ja...@apache.org>
Authored: Tue Feb 23 16:52:05 2016 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Tue Feb 23 16:52:05 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/execute/AggregatePlan.java | 6 ++++--
.../java/org/apache/phoenix/execute/BaseQueryPlan.java | 11 +++++++++++
.../main/java/org/apache/phoenix/execute/ScanPlan.java | 7 +++++--
3 files changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e3a62026/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
index 73a995c..a8a7dc5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
@@ -35,6 +35,7 @@ import org.apache.phoenix.expression.OrderByExpression;
import org.apache.phoenix.expression.RowKeyExpression;
import org.apache.phoenix.expression.aggregator.Aggregators;
import org.apache.phoenix.iterate.AggregatingResultIterator;
+import org.apache.phoenix.iterate.BaseResultIterators;
import org.apache.phoenix.iterate.ConcatResultIterator;
import org.apache.phoenix.iterate.DistinctAggregatingResultIterator;
import org.apache.phoenix.iterate.FilterAggregatingResultIterator;
@@ -48,7 +49,6 @@ import org.apache.phoenix.iterate.ParallelIterators;
import org.apache.phoenix.iterate.ParallelScanGrouper;
import org.apache.phoenix.iterate.PeekingResultIterator;
import org.apache.phoenix.iterate.ResultIterator;
-import org.apache.phoenix.iterate.ResultIterators;
import org.apache.phoenix.iterate.SequenceResultIterator;
import org.apache.phoenix.iterate.SerialIterators;
import org.apache.phoenix.iterate.SpoolingResultIterator;
@@ -197,12 +197,14 @@ public class AggregatePlan extends BaseQueryPlan {
context.getScan().setAttribute(BaseScannerRegionObserver.GROUP_BY_LIMIT, PInteger.INSTANCE.toBytes(limit));
}
}
- ResultIterators iterators = statement.getHint().hasHint(HintNode.Hint.SERIAL) ?
+ BaseResultIterators iterators = statement.getHint().hasHint(HintNode.Hint.SERIAL) ?
new SerialIterators(this, null, wrapParallelIteratorFactory(), scanGrouper) :
new ParallelIterators(this, null, wrapParallelIteratorFactory());
splits = iterators.getSplits();
scans = iterators.getScans();
+ estimatedSize = iterators.getEstimatedByteCount();
+ estimatedRows = iterators.getEstimatedRowCount();
AggregatingResultIterator aggResultIterator;
// No need to merge sort for ungrouped aggregation
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e3a62026/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 2aef7f7..0ee70ba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -108,6 +108,9 @@ public abstract class BaseQueryPlan implements QueryPlan {
* immediately before creating the ResultIterator.
*/
protected final Expression dynamicFilter;
+ protected Long estimatedRows;
+ protected Long estimatedSize;
+
protected BaseQueryPlan(
StatementContext context, FilterableStatement statement, TableRef table,
@@ -127,6 +130,14 @@ public abstract class BaseQueryPlan implements QueryPlan {
this.dynamicFilter = dynamicFilter;
}
+ public Long getEstimatedRowCount() {
+ return this.estimatedRows;
+ }
+
+ public Long getEstimatedByteCount() {
+ return this.estimatedSize;
+ }
+
@Override
public Operation getOperation() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e3a62026/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index d51e6c8..4a5cb83 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -31,6 +31,7 @@ import org.apache.phoenix.compile.StatementContext;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.ScanRegionObserver;
import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.iterate.BaseResultIterators;
import org.apache.phoenix.iterate.ChunkedResultIterator;
import org.apache.phoenix.iterate.ConcatResultIterator;
import org.apache.phoenix.iterate.LimitingResultIterator;
@@ -40,7 +41,6 @@ import org.apache.phoenix.iterate.ParallelIteratorFactory;
import org.apache.phoenix.iterate.ParallelIterators;
import org.apache.phoenix.iterate.ParallelScanGrouper;
import org.apache.phoenix.iterate.ResultIterator;
-import org.apache.phoenix.iterate.ResultIterators;
import org.apache.phoenix.iterate.RoundRobinResultIterator;
import org.apache.phoenix.iterate.SequenceResultIterator;
import org.apache.phoenix.iterate.SerialIterators;
@@ -195,7 +195,7 @@ public class ScanPlan extends BaseQueryPlan {
boolean isOrdered = !orderBy.getOrderByExpressions().isEmpty();
boolean isSerial = isSerial(context, statement, tableRef, orderBy, limit, allowPageFilter);
Integer perScanLimit = !allowPageFilter || isOrdered ? null : limit;
- ResultIterators iterators;
+ BaseResultIterators iterators;
if (isSerial) {
iterators = new SerialIterators(this, perScanLimit, parallelIteratorFactory, scanGrouper);
} else {
@@ -203,6 +203,9 @@ public class ScanPlan extends BaseQueryPlan {
}
splits = iterators.getSplits();
scans = iterators.getScans();
+ estimatedSize = iterators.getEstimatedByteCount();
+ estimatedRows = iterators.getEstimatedRowCount();
+
if (isOrdered) {
scanner = new MergeSortTopNResultIterator(iterators, limit, orderBy.getOrderByExpressions());
} else {
[07/12] phoenix git commit: PHOENIX-2713 Backward compatibility fails
with CNF exception with 4.7 client and server when upgraded from 4.6
Posted by ma...@apache.org.
PHOENIX-2713 Backward compatibility fails with CNF exception with 4.7 client and server when upgraded from 4.6
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1f7b47a5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1f7b47a5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1f7b47a5
Branch: refs/heads/calcite
Commit: 1f7b47a5e75fe5f6014b4dcdeb4637d05fd12a43
Parents: 05e6e2a
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Feb 24 11:48:16 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Feb 24 12:01:21 2016 -0800
----------------------------------------------------------------------
.../phoenix/coprocessor/MetaDataProtocol.java | 2 +-
.../query/ConnectionQueryServicesImpl.java | 35 ++++++++++++++++----
2 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f7b47a5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index a704e22..fb9d228 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -73,7 +73,7 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0 = MIN_TABLE_TIMESTAMP + 7;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0 = MIN_TABLE_TIMESTAMP + 8;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0 = MIN_TABLE_TIMESTAMP + 9;
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 13;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 14;
// MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0;
// TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f7b47a5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index d55ab30..d27a4bc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2370,17 +2370,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// Add these columns one at a time, each with different timestamps so that if folks have
// run the upgrade code already for a snapshot, we'll still enter this block (and do the
// parts we haven't yet done).
- metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
+ metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4,
PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName());
// Drop old stats table so that new stats table is created
metaConnection = dropStatsTable(metaConnection,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3);
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
+ PLong.INSTANCE.getSqlTypeName());
- setImmutableTableIndexesImmutable(metaConnection);
+ metaConnection = setImmutableTableIndexesImmutable(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
+ Properties props = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0));
+ metaConnection = new PhoenixConnection(metaConnection, ConnectionQueryServicesImpl.this, props);
// that already have cached data.
clearCache();
}
@@ -2490,7 +2493,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
* @param metaConnection connection over which to run the upgrade
* @throws SQLException
*/
- private static void setImmutableTableIndexesImmutable(PhoenixConnection metaConnection) throws SQLException {
+ private PhoenixConnection setImmutableTableIndexesImmutable(PhoenixConnection oldMetaConnection, long timestamp) throws SQLException {
+ SQLException sqlE = null;
+ Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
+ PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props);
boolean autoCommit = metaConnection.getAutoCommit();
try {
metaConnection.setAutoCommit(true);
@@ -2507,9 +2514,25 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
"WHERE A.COLUMN_FAMILY IS NULL AND\n" +
" B.COLUMN_FAMILY IS NOT NULL AND\n" +
" A.IMMUTABLE_ROWS = TRUE");
+ } catch (SQLException e) {
+ logger.warn("exception during upgrading stats table:" + e);
+ sqlE = e;
} finally {
- metaConnection.setAutoCommit(autoCommit);
+ try {
+ metaConnection.setAutoCommit(autoCommit);
+ oldMetaConnection.close();
+ } catch (SQLException e) {
+ if (sqlE != null) {
+ sqlE.setNextException(e);
+ } else {
+ sqlE = e;
+ }
+ }
+ if (sqlE != null) {
+ throw sqlE;
+ }
}
+ return metaConnection;
}
private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, long timestamp)
[03/12] phoenix git commit: Revert "PHOENIX-2706 Implement
client-side mechanism to know if stats are enabled"
Posted by ma...@apache.org.
Revert "PHOENIX-2706 Implement client-side mechanism to know if stats are enabled"
This reverts commit a1e6ae44be757b3fc5c4192ad55ecd8f3c03c01a.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dbc9ee9d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dbc9ee9d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dbc9ee9d
Branch: refs/heads/calcite
Commit: dbc9ee9dfe9e168c45ad279f8478c59f0882240c
Parents: da51f46
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Feb 23 09:36:53 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Feb 23 09:36:53 2016 -0800
----------------------------------------------------------------------
.../StatisticsCollectionRunTrackerIT.java | 10 ++--
.../org/apache/phoenix/end2end/IndexToolIT.java | 3 +-
.../phoenix/end2end/MutableIndexToolIT.java | 1 -
.../end2end/StatsCollectionDisabledIT.java | 19 ++-----
.../phoenix/end2end/StatsCollectorIT.java | 3 --
.../coprocessor/MetaDataEndpointImpl.java | 12 ++++-
.../UngroupedAggregateRegionObserver.java | 27 +++++-----
.../phoenix/iterate/BaseResultIterators.java | 25 +--------
.../phoenix/query/ConnectionQueryServices.java | 7 +--
.../query/ConnectionQueryServicesImpl.java | 25 ++-------
.../query/ConnectionlessQueryServicesImpl.java | 18 +------
.../query/DelegateConnectionQueryServices.java | 10 ----
.../org/apache/phoenix/query/QueryServices.java | 1 -
.../phoenix/query/QueryServicesOptions.java | 7 ---
.../apache/phoenix/schema/MetaDataClient.java | 2 -
.../schema/stats/NoOpStatisticsCollector.java | 3 ++
.../stats/StatisticsCollectionRunTracker.java | 7 +--
.../phoenix/schema/stats/StatisticsUtil.java | 16 ------
.../org/apache/phoenix/util/MetaDataUtil.java | 55 +++++++-------------
.../phoenix/query/QueryServicesTestImpl.java | 2 -
.../apache/phoenix/util/MetaDataUtilTest.java | 33 ------------
21 files changed, 71 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
index 345400e..bf567f0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/coprocessor/StatisticsCollectionRunTrackerIT.java
@@ -17,7 +17,8 @@
*/
package org.apache.phoenix.coprocessor;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_SKIPPED;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -37,6 +38,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -103,8 +105,7 @@ public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManaged
HRegionInfo regionInfo = createTableAndGetRegion(tableName);
// simulate stats collection via major compaction by marking the region as compacting in the tracker
markRegionAsCompacting(regionInfo);
- long returnValue = runUpdateStats(tableName);
- assertTrue("Update stats should have been skipped", returnValue >= UPDATE_STATS_SKIPPED);
+ Assert.assertEquals("Row count didn't match", COMPACTION_UPDATE_STATS_ROW_COUNT, runUpdateStats(tableName));
StatisticsCollectionRunTracker tracker =
StatisticsCollectionRunTracker.getInstance(new Configuration());
// assert that the tracker state was cleared.
@@ -116,7 +117,8 @@ public class StatisticsCollectionRunTrackerIT extends BaseOwnClusterHBaseManaged
String tableName = "testUpdateStatsPreventsAnotherUpdateStatsFromRunning".toUpperCase();
HRegionInfo regionInfo = createTableAndGetRegion(tableName);
markRunningUpdateStats(regionInfo);
- assertTrue("Update stats should have been skipped", runUpdateStats(tableName) >= UPDATE_STATS_SKIPPED);
+ Assert.assertEquals("Row count didn't match", CONCURRENT_UPDATE_STATS_ROW_COUNT,
+ runUpdateStats(tableName));
// assert that running the concurrent and race-losing update stats didn't clear the region
// from the tracker. If the method returned true it means the tracker was still tracking
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index fe95470..aba9c11 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -108,8 +108,7 @@ public class IndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String fullTableName = SchemaUtil.getTableName(schemaName, dataTable);
final String indxTable = String.format("%s_%s", dataTable, "INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
- props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
+ props.setProperty(QueryServices.TRANSACTIONS_ENABLED, "true");
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
index 8125007..0791479 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
@@ -61,7 +61,6 @@ public class MutableIndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String dataTable = "DATA_TABLE5";
final String indxTable = String.format("%s_%s",dataTable,"INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
index c3cdbc0..a92a665 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
@@ -17,27 +17,24 @@
*/
package org.apache.phoenix.end2end;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.Properties;
-import org.apache.phoenix.jdbc.PhoenixConnection;
+import com.google.common.collect.Maps;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.common.collect.Maps;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertFalse;
/**
* Verifies that statistics are not collected if they are disabled via a setting
@@ -57,21 +54,15 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
public void testStatisticsAreNotWritten() throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
- assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
Statement stmt = conn.createStatement();
- stmt.execute("CREATE TABLE T1 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR) SALT_BUCKETS=3");
+ stmt.execute("CREATE TABLE T1 (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR)");
stmt.execute("UPSERT INTO T1 VALUES (1, 'NAME1')");
stmt.execute("UPSERT INTO T1 VALUES (2, 'NAME2')");
stmt.execute("UPSERT INTO T1 VALUES (3, 'NAME3')");
conn.commit();
stmt.execute("UPDATE STATISTICS T1");
- assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
ResultSet rs = stmt.executeQuery("SELECT * FROM SYSTEM.STATS");
assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("SELECT count(*) FROM T1");
- assertTrue(rs.next());
- assertEquals(3,rs.getInt(1));
- assertFalse(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
rs.close();
stmt.close();
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index 6cd8e13..e72f41f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -94,7 +94,6 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
// props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
conn = DriverManager.getConnection(getUrl(), props);
- assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
conn.createStatement().execute(
"CREATE TABLE " + fullTableName +" ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC))"
@@ -105,7 +104,6 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// CAll the update statistics query here. If already major compaction has run this will not get executed.
stmt = conn.prepareStatement("UPDATE STATISTICS " + tableName);
stmt.execute();
- assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
stmt = upsertStmt(conn, tableName);
stmt.setString(1, "z");
s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
@@ -122,7 +120,6 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
stmt.execute();
rs = conn.createStatement().executeQuery("SELECT k FROM " + tableName);
assertTrue(rs.next());
- assertTrue(conn.unwrap(PhoenixConnection.class).getQueryServices().areStatsEnabled());
conn.close();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 411ce03..ba7eb39 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -138,6 +138,7 @@ import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -2716,7 +2717,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
RpcCallback<GetVersionResponse> done) {
GetVersionResponse.Builder builder = GetVersionResponse.newBuilder();
- long version = MetaDataUtil.encodeVersion(env.getHBaseVersion(), env.getConfiguration());
+ // The first 3 bytes of the long is used to encoding the HBase version as major.minor.patch.
+ // The next 4 bytes of the value is used to encode the Phoenix version as major.minor.patch.
+ long version = MetaDataUtil.encodeHBaseAndPhoenixVersions(this.env.getHBaseVersion());
+
+ // The last byte is used to communicate whether or not mutable secondary indexing
+ // was configured properly.
+ version =
+ MetaDataUtil.encodeHasIndexWALCodec(version,
+ IndexManagementUtil.isWALEditCodecSet(this.env.getConfiguration()));
+
builder.setVersion(version);
done.run(builder.build());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 91ec20e..77b8b3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -22,9 +22,8 @@ import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_DISABLED;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_RUN;
-import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.UPDATE_STATS_SKIPPED;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
+import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -60,6 +59,7 @@ import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.exception.DataExceedsCapacityException;
@@ -90,7 +90,6 @@ import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.stats.StatisticsCollector;
import org.apache.phoenix.schema.stats.StatisticsCollectorFactory;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
import org.apache.phoenix.schema.types.PBinary;
import org.apache.phoenix.schema.types.PChar;
@@ -609,6 +608,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
InternalScanner internalScanner = scanner;
if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
try {
+ Pair<HRegionInfo, HRegionInfo> mergeRegions = null;
long clientTimeStamp = TimeKeeper.SYSTEM.getCurrentTime();
StatisticsCollector stats = StatisticsCollectorFactory.createStatisticsCollector(
c.getEnvironment(), table.getNameAsString(), clientTimeStamp,
@@ -643,23 +643,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
if (asyncBytes != null) {
async = Bytes.toBoolean(asyncBytes);
}
- long returnValue = UPDATE_STATS_RUN; // in case of async, we report 1 as number of rows updated
- boolean statsEnabled = StatisticsUtil.isStatsEnabled(config);
+ long rowCount = 0; // in case of async, we report 0 as number of rows updated
StatisticsCollectionRunTracker statsRunTracker =
StatisticsCollectionRunTracker.getInstance(config);
- boolean runUpdateStats = statsEnabled && statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
+ boolean runUpdateStats = statsRunTracker.addUpdateStatsCommandRegion(region.getRegionInfo());
if (runUpdateStats) {
if (!async) {
- returnValue = callable.call();
+ rowCount = callable.call();
} else {
statsRunTracker.runTask(callable);
}
} else {
- returnValue = statsEnabled ? UPDATE_STATS_SKIPPED : UPDATE_STATS_DISABLED;
- logger.info("UPDATE STATISTICS didn't run because " + (statsEnabled ? " another UPDATE STATISTICS command was already running on the region " : " stats are disabled ")
+ rowCount = CONCURRENT_UPDATE_STATS_ROW_COUNT;
+ logger.info("UPDATE STATISTICS didn't run because another UPDATE STATISTICS command was already running on the region "
+ region.getRegionInfo().getRegionNameAsString());
}
- byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(returnValue));
+ byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
final KeyValue aggKeyValue =
KeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
@@ -729,22 +728,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
region.startRegionOperation();
boolean hasMore = false;
boolean noErrors = false;
- long rowCount = 0;
boolean compactionRunning = areStatsBeingCollectedViaCompaction();
+ long rowCount = 0;
try {
if (!compactionRunning) {
synchronized (innerScanner) {
do {
List<Cell> results = new ArrayList<Cell>();
hasMore = innerScanner.nextRaw(results);
- rowCount++;
stats.collectStatistics(results);
+ rowCount++;
compactionRunning = areStatsBeingCollectedViaCompaction();
} while (hasMore && !compactionRunning);
noErrors = true;
}
}
- return compactionRunning ? UPDATE_STATS_SKIPPED : UPDATE_STATS_RUN;
+ return compactionRunning ? COMPACTION_UPDATE_STATS_ROW_COUNT : rowCount;
} catch (IOException e) {
logger.error("IOException in update stats: " + Throwables.getStackTraceAsString(e));
throw e;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 8d1a365..fc3edbe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -122,7 +122,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
private long estimatedRows;
private long estimatedSize;
- private boolean areStatsEnabled;
static final Function<HRegionLocation, KeyRange> TO_KEY_RANGE = new Function<HRegionLocation, KeyRange>() {
@Override
@@ -370,11 +369,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return scans;
}
- // TODO: add to ResultIterators and QueryPlan interfaces?
- public boolean areStatsEnabled() {
- return this.areStatsEnabled;
- }
-
private static List<byte[]> toBoundaries(List<HRegionLocation> regionLocations) {
int nBoundaries = regionLocations.size() - 1;
List<byte[]> ranges = Lists.newArrayListWithExpectedSize(nBoundaries);
@@ -530,17 +524,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY;
List<Scan> scans = Lists.newArrayListWithExpectedSize(estGuidepostsPerRegion);
ImmutableBytesWritable guidePosts = gps.getGuidePosts();
- // If we have any guideposts, then we can definitely say that stats are enabled.
- // If we have no guideposts, though, we cannot assume that stats are disabled,
- // as the table may just be too small to have them.
- if (guidePosts.getLength() > 0) {
- areStatsEnabled = true;
- // It's possible that the server was bounced and stats have changed
- // to become enabled without a client bounce.
- this.context.getConnection().getQueryServices().setStatsEnabled(true);
- } else {
- areStatsEnabled = this.context.getConnection().getQueryServices().areStatsEnabled();
- }
ByteArrayInputStream stream = null;
DataInput input = null;
PrefixByteDecoder decoder = null;
@@ -868,18 +851,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
@Override
public void explain(List<String> planSteps) {
- ConnectionQueryServices services = context.getConnection().getQueryServices();
- boolean displayChunkCount = services.getProps().getBoolean(
+ boolean displayChunkCount = context.getConnection().getQueryServices().getProps().getBoolean(
QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB,
QueryServicesOptions.DEFAULT_EXPLAIN_CHUNK_COUNT);
- boolean displayRowCount = services.getProps().getBoolean(
- QueryServices.EXPLAIN_ROW_COUNT_ATTRIB,
- QueryServicesOptions.DEFAULT_EXPLAIN_ROW_COUNT);
StringBuilder buf = new StringBuilder();
buf.append("CLIENT ");
if (displayChunkCount) {
buf.append(this.splits.size()).append("-CHUNK ");
- if (displayRowCount && areStatsEnabled) {
+ if (estimatedRows > 0) {
buf.append(estimatedRows).append(" ROWS ");
buf.append(estimatedSize).append(" BYTES ");
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 471562a..b5f1f85 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -23,6 +23,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import co.cask.tephra.TransactionSystemClient;
+
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -44,8 +46,6 @@ import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.stats.PTableStats;
-import co.cask.tephra.TransactionSystemClient;
-
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
public static final int INITIAL_META_DATA_TABLE_CAPACITY = 100;
@@ -123,7 +123,4 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
TransactionSystemClient getTransactionSystemClient();
public long getRenewLeaseThresholdMilliSeconds();
public boolean isRenewingLeasesEnabled();
-
- public boolean areStatsEnabled();
- public void setStatsEnabled(boolean statsEnabled);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b428eb9..d55ab30 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -241,7 +241,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private TransactionServiceClient txServiceClient;
private volatile boolean initialized;
private volatile int nSequenceSaltBuckets;
- private volatile boolean areStatsEnabled;
// writes guarded by "this"
private volatile boolean closed;
@@ -1109,7 +1108,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void checkClientServerCompatibility() throws SQLException {
StringBuilder buf = new StringBuilder("The following servers require an updated " + QueryConstants.DEFAULT_COPROCESS_PATH + " to be put in the classpath of HBase: ");
boolean isIncompatible = false;
- boolean areStatsEnabled = false;
int minHBaseVersion = Integer.MAX_VALUE;
try {
List<HRegionLocation> locations = this.getAllTableRegions(SYSTEM_CATALOG_NAME_BYTES);
@@ -1143,23 +1141,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
});
for (Map.Entry<byte[],Long> result : results.entrySet()) {
// This is the "phoenix.jar" is in-place, but server is out-of-sync with client case.
- long version = result.getValue();
- // Set stats as being enabled if enabled on *any* region server (though it should
- // really match across all regions servers).
- areStatsEnabled |= MetaDataUtil.decodeStatsEnabled(version);
- if (!isCompatible(version)) {
+ if (!isCompatible(result.getValue())) {
isIncompatible = true;
HRegionLocation name = regionMap.get(result.getKey());
buf.append(name);
buf.append(';');
}
- hasIndexWALCodec &= hasIndexWALCodec(version);
- if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(version)) {
- minHBaseVersion = MetaDataUtil.decodeHBaseVersion(version);
+ hasIndexWALCodec &= hasIndexWALCodec(result.getValue());
+ if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
+ minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
}
}
lowestClusterHBaseVersion = minHBaseVersion;
- this.areStatsEnabled = areStatsEnabled;
} catch (SQLException e) {
throw e;
} catch (Throwable t) {
@@ -3386,14 +3379,4 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
- @Override
- public boolean areStatsEnabled() {
- return areStatsEnabled;
- }
-
- @Override
- public void setStatsEnabled(boolean statsEnabled) {
- this.areStatsEnabled = statsEnabled;
- }
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 453d04f..b4bbe1f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -76,7 +76,6 @@ import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.stats.PTableStats;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -107,8 +106,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private final Map<SequenceKey, SequenceInfo> sequenceMap = Maps.newHashMap();
private final String userName;
private final TransactionSystemClient txSystemClient;
- private final KeyValueBuilder kvBuilder;
- private volatile boolean areStatsEnabled;
+ private KeyValueBuilder kvBuilder;
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
@@ -138,8 +136,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
TransactionManager txnManager = new TransactionManager(config);
this.txSystemClient = new InMemoryTxSystemClient(txnManager);
- // Just check the properties on the client side (instead of normally the server side)
- this.areStatsEnabled = StatisticsUtil.isStatsEnabled(config);
}
private PMetaData newEmptyMetaData() {
@@ -531,7 +527,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return txSystemClient;
}
- @Override
public MetaDataMutationResult createFunction(List<Mutation> functionData, PFunction function, boolean temporary)
throws SQLException {
return new MetaDataMutationResult(MutationCode.FUNCTION_NOT_FOUND, 0l, null);
@@ -583,7 +578,6 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return false;
}
- @Override
public HRegionLocation getTableRegionLocation(byte[] tableName, byte[] row) throws SQLException {
List<HRegionLocation> regions = tableSplits.get(Bytes.toString(tableName));
if (regions != null) {
@@ -598,14 +592,4 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
new HRegionInfo(TableName.valueOf(tableName), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW),
SERVER_NAME, -1);
}
-
- @Override
- public boolean areStatsEnabled() {
- return areStatsEnabled;
- }
-
- @Override
- public void setStatsEnabled(boolean statsEnabled) {
- this.areStatsEnabled = statsEnabled;
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 2d0f677..4c7446b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -310,14 +310,4 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
throws SQLException {
return getDelegate().getTableRegionLocation(tableName, row);
}
-
- @Override
- public boolean areStatsEnabled() {
- return getDelegate().areStatsEnabled();
- }
-
- @Override
- public void setStatsEnabled(boolean statsEnabled) {
- getDelegate().setStatsEnabled(statsEnabled);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 32b4fc0..1efcd8c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -162,7 +162,6 @@ public interface QueryServices extends SQLCloseable {
public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets";
public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority";
public static final String EXPLAIN_CHUNK_COUNT_ATTRIB = "phoenix.explain.displayChunkCount";
- public static final String EXPLAIN_ROW_COUNT_ATTRIB = "phoenix.explain.displayRowCount";
public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 3e0ffe1..27c5693 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -29,7 +29,6 @@ import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB
import static org.apache.phoenix.query.QueryServices.DELAY_FOR_SCHEMA_UPDATE_CHECK;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.EXPLAIN_ROW_COUNT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GLOBAL_METRICS_ENABLED;
@@ -197,7 +196,6 @@ public class QueryServicesOptions {
*/
public static final int DEFAULT_COPROCESSOR_PRIORITY = Coprocessor.PRIORITY_SYSTEM/2 + Coprocessor.PRIORITY_USER/2; // Divide individually to prevent any overflow
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = true;
- public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = true;
public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
@@ -575,11 +573,6 @@ public class QueryServicesOptions {
return this;
}
- public QueryServicesOptions setExplainRowCount(boolean showRowCount) {
- config.setBoolean(EXPLAIN_ROW_COUNT_ATTRIB, showRowCount);
- return this;
- }
-
public QueryServicesOptions setAllowOnlineSchemaUpdate(boolean allow) {
config.setBoolean(ALLOW_ONLINE_TABLE_SCHEMA_UPDATE, allow);
return this;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index eee42ea..7f3f850 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -182,7 +182,6 @@ import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.stats.PTableStats;
-import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
@@ -956,7 +955,6 @@ public class MetaDataClient {
}
MutationState mutationState = plan.execute();
rowCount = mutationState.getUpdateCount();
- this.getConnection().getQueryServices().setStatsEnabled(rowCount != StatisticsCollectionRunTracker.UPDATE_STATS_DISABLED);
}
/*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index 6c73f16..1063229 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -21,10 +21,13 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
index 6413e37..4ed3325 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectionRunTracker.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -44,9 +45,9 @@ public class StatisticsCollectionRunTracker {
.newSetFromMap(new ConcurrentHashMap<HRegionInfo, Boolean>());
private final ExecutorService executor;
- public static final long UPDATE_STATS_RUN = 1L;
- public static final long UPDATE_STATS_SKIPPED = 100000L;
- public static final long UPDATE_STATS_DISABLED = 0;
+ // Constants added for testing purposes
+ public static final long CONCURRENT_UPDATE_STATS_ROW_COUNT = -100l;
+ public static final long COMPACTION_UPDATE_STATS_ROW_COUNT = -200l;
public static StatisticsCollectionRunTracker getInstance(Configuration config) {
StatisticsCollectionRunTracker result = INSTANCE;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 9dd72e8..5b47104 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
@@ -41,8 +40,6 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
@@ -223,17 +220,4 @@ public class StatisticsUtil {
ptr.set(row, gpOffset, row.length - gpOffset);
return ByteUtil.copyKeyBytesIfNecessary(ptr);
}
-
- public static boolean isStatsEnabled(Configuration conf) {
- if (conf.getBoolean(QueryServices.STATS_ENABLED_ATTRIB, true)) {
- if (conf.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) != 1) {
- if (conf.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) < HConstants.DEFAULT_MAX_FILE_SIZE) {
- return true;
- }
- }
- }
- return false;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 5ee3c0a..6d5903d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -57,7 +56,6 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PLong;
@@ -105,25 +103,26 @@ public class MetaDataUtil {
// The second byte in int would be the major version, 3rd byte minor version, and 4th byte
// patch version.
public static int decodePhoenixVersion(long version) {
- return (int) ((version << Byte.SIZE * 4) >>> Byte.SIZE * 5);
+ return (int) ((version << Byte.SIZE * 3) >>> Byte.SIZE * 4);
+ }
+
+ // TODO: generalize this to use two bytes to return a SQL error code instead
+ public static long encodeHasIndexWALCodec(long version, boolean isValid) {
+ if (!isValid) {
+ return version | 1;
+ }
+ return version;
}
public static boolean decodeHasIndexWALCodec(long version) {
return (version & 0xF) == 0;
}
- // Given the encoded integer representing the phoenix version in the encoded version value.
- // The second byte in int would be the major version, 3rd byte minor version, and 4th byte
- // patch version.
- public static boolean decodeStatsEnabled(long version) {
- return ((int) ((version << Byte.SIZE * 3) >>> Byte.SIZE * 7) & 0x1) != 0;
- }
-
// Given the encoded integer representing the client hbase version in the encoded version value.
// The second byte in int would be the major version, 3rd byte minor version, and 4th byte
// patch version.
public static int decodeHBaseVersion(long version) {
- return (int) (version >>> (Byte.SIZE * 5));
+ return (int) (version >>> Byte.SIZE * 5);
}
public static String decodeHBaseVersionAsString(int version) {
@@ -133,33 +132,17 @@ public class MetaDataUtil {
return major + "." + minor + "." + patch;
}
- // The first 3 bytes of the long is used to encoding the HBase version as major.minor.patch.
- // The next 4 bytes of the value is used to encode the Phoenix version as major.minor.patch.
- /**
- * Encode HBase and Phoenix version along with some server-side config information such
- * as whether WAL codec is installed (necessary for non transactional, mutable secondar
- * indexing), and whether stats are enabled.
- * @param env RegionCoprocessorEnvironment to access HBase version and Configuration.
- * @return long value sent back during initialization of a cluster connection.
- */
- public static long encodeVersion(String hbaseVersionStr, Configuration config) {
- long hbaseVersion = VersionUtil.encodeVersion(hbaseVersionStr);
- long statsEnabled = StatisticsUtil.isStatsEnabled(config) ? 1 : 0;
- long phoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ public static int encodePhoenixVersion() {
+ return VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
MetaDataProtocol.PHOENIX_PATCH_NUMBER);
- long walCodec = IndexManagementUtil.isWALEditCodecSet(config) ? 0 : 1;
- long version =
- // Encode HBase major, minor, patch version
- (hbaseVersion << (Byte.SIZE * 5))
- // Encode if stats are enabled on the server side
- | (statsEnabled << (Byte.SIZE * 4))
- // Encode Phoenix major, minor, patch version
- | (phoenixVersion << (Byte.SIZE * 1))
- // Encode whether or not non transactional, mutable secondary indexing was configured properly.
- | walCodec;
- return version;
}
-
+
+ public static long encodeHBaseAndPhoenixVersions(String hbaseVersion) {
+ return (((long) VersionUtil.encodeVersion(hbaseVersion)) << (Byte.SIZE * 5)) |
+ (((long) VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
+ MetaDataProtocol.PHOENIX_PATCH_NUMBER)) << (Byte.SIZE * 1));
+ }
+
public static void getTenantIdAndSchemaAndTableName(List<Mutation> tableMetadata, byte[][] rowKeyMetaData) {
Mutation m = getTableHeaderRow(tableMetadata);
getVarChars(m.getRow(), 3, rowKeyMetaData);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 6ae655c..29a7001 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -54,7 +54,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = false; // TODO: update explain plans in test and set to true
- public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = false; // TODO: update explain plans in test and set to true
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
@@ -81,7 +80,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static QueryServicesOptions getDefaultServicesOptions() {
return withDefaults()
.setExplainChunkCount(DEFAULT_EXPLAIN_CHUNK_COUNT)
- .setExplainRowCount(DEFAULT_EXPLAIN_ROW_COUNT)
.setSequenceSaltBuckets(DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS)
.setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dbc9ee9d/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
index 7ffc054..1e06379 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/MetaDataUtilTest.java
@@ -21,19 +21,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.query.HBaseFactoryProvider;
-import org.apache.phoenix.query.QueryServices;
import org.junit.Test;
@@ -62,35 +58,6 @@ public class MetaDataUtilTest {
assertFalse(MetaDataUtil.areClientAndServerCompatible(VersionUtil.encodeVersion(3,1,10), 3, 5));
}
- @Test
- public void testEncodeDecode() {
- String hbaseVersionStr = "0.98.14";
- Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
- config.setBoolean(QueryServices.STATS_ENABLED_ATTRIB, false);
-
- long version = MetaDataUtil.encodeVersion(hbaseVersionStr, config);
- int hbaseVersion = MetaDataUtil.decodeHBaseVersion(version);
- int expectedHBaseVersion = VersionUtil.encodeVersion(0, 98, 14);
- assertEquals(expectedHBaseVersion, hbaseVersion);
- boolean areStatsEnabled = MetaDataUtil.decodeStatsEnabled(version);
- assertFalse(areStatsEnabled);
- int phoenixVersion = MetaDataUtil.decodePhoenixVersion(version);
- int expectedPhoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
- MetaDataProtocol.PHOENIX_PATCH_NUMBER);
- assertEquals(expectedPhoenixVersion, phoenixVersion);
-
- config.setBoolean(QueryServices.STATS_ENABLED_ATTRIB, true);
- version = MetaDataUtil.encodeVersion(hbaseVersionStr, config);
- hbaseVersion = MetaDataUtil.decodeHBaseVersion(version);
- expectedHBaseVersion = VersionUtil.encodeVersion(0, 98, 14);
- assertEquals(expectedHBaseVersion, hbaseVersion);
- areStatsEnabled = MetaDataUtil.decodeStatsEnabled(version);
- assertTrue(areStatsEnabled);
- phoenixVersion = MetaDataUtil.decodePhoenixVersion(version);
- expectedPhoenixVersion = VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, MetaDataProtocol.PHOENIX_MINOR_VERSION,
- MetaDataProtocol.PHOENIX_PATCH_NUMBER);
- assertEquals(expectedPhoenixVersion, phoenixVersion);
- }
/**
* Ensure it supports {@link GenericKeyValueBuilder}
* @throws Exception on failure
[04/12] phoenix git commit: PHOENIX-2707 Differentiate between a
table+family have zero guideposts from not having collected guideposts
Posted by ma...@apache.org.
PHOENIX-2707 Differentiate between a table+family have zero guideposts from not having collected guideposts
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c25f8879
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c25f8879
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c25f8879
Branch: refs/heads/calcite
Commit: c25f88791c86cd65ea21810406eec199403d78aa
Parents: dbc9ee9
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Feb 23 14:53:11 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Feb 23 14:53:11 2016 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/IndexToolIT.java | 3 +-
.../phoenix/end2end/MutableIndexToolIT.java | 1 +
.../end2end/StatsCollectionDisabledIT.java | 21 ++-
.../phoenix/end2end/StatsCollectorIT.java | 57 +++++++-
.../phoenix/iterate/BaseResultIterators.java | 140 +++++++++++--------
.../apache/phoenix/query/QueryConstants.java | 2 +-
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 7 +
.../stats/DefaultStatisticsCollector.java | 33 ++---
.../phoenix/schema/stats/GuidePostsInfo.java | 2 +-
.../schema/stats/GuidePostsInfoBuilder.java | 6 -
.../phoenix/schema/stats/PTableStatsImpl.java | 5 +-
.../phoenix/schema/stats/StatisticsUtil.java | 23 +--
.../phoenix/schema/stats/StatisticsWriter.java | 60 ++++----
.../apache/phoenix/util/PrefixByteCodec.java | 37 ++---
.../phoenix/query/QueryServicesTestImpl.java | 2 +
16 files changed, 246 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index aba9c11..fe95470 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -108,7 +108,8 @@ public class IndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String fullTableName = SchemaUtil.getTableName(schemaName, dataTable);
final String indxTable = String.format("%s_%s", dataTable, "INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(QueryServices.TRANSACTIONS_ENABLED, "true");
+ props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
index 0791479..8125007 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MutableIndexToolIT.java
@@ -61,6 +61,7 @@ public class MutableIndexToolIT extends BaseOwnClusterHBaseManagedTimeIT {
final String dataTable = "DATA_TABLE5";
final String indxTable = String.format("%s_%s",dataTable,"INDX");
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.FALSE.toString());
Connection conn = DriverManager.getConnection(getUrl(), props);
Statement stmt = conn.createStatement();
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
index a92a665..54ffa7c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectionDisabledIT.java
@@ -17,24 +17,26 @@
*/
package org.apache.phoenix.end2end;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.Properties;
-import com.google.common.collect.Maps;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertFalse;
+import com.google.common.collect.Maps;
/**
* Verifies that statistics are not collected if they are disabled via a setting
@@ -43,10 +45,12 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(5);
// Must update config before starting server
props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
props.put(QueryServices.STATS_ENABLED_ATTRIB, Boolean.toString(false));
+ props.put(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
+ props.put(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString());
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -65,6 +69,11 @@ public class StatsCollectionDisabledIT extends StatsCollectorAbstractIT {
assertFalse(rs.next());
rs.close();
stmt.close();
- conn.close();
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM T1");
+ String explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER T1",
+ explainPlan);
+ conn.close();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index e72f41f..bc575fd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -44,6 +44,7 @@ import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
@@ -65,10 +66,11 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(10);
// Must update config before starting server
props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
props.put(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
+ props.put(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString());
props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(1024));
props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
@@ -86,6 +88,55 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
}
@Test
+ public void testUpdateEmptyStats() throws Exception {
+ Connection conn;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName +" ( k CHAR(1) PRIMARY KEY )" + tableDDLOptions);
+ conn.createStatement().execute("UPDATE STATISTICS " + tableName);
+ ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName);
+ String explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 1-CHUNK 0 ROWS 0 BYTES PARALLEL 1-WAY FULL SCAN OVER " + fullTableName + "\n" +
+ " SERVER FILTER BY FIRST KEY ONLY",
+ explainPlan);
+ conn.close();
+ }
+
+ @Test
+ public void testSomeUpdateEmptyStats() throws Exception {
+ Connection conn;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String fullTableName = this.fullTableName + "_SALTED";
+ // props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullTableName +" ( k VARCHAR PRIMARY KEY, a.v1 VARCHAR, b.v2 VARCHAR ) " + tableDDLOptions + (tableDDLOptions.isEmpty() ? "" : ",") + "SALT_BUCKETS = 3");
+ conn.createStatement().execute("UPSERT INTO " + fullTableName + "(k,v1) VALUES('a','123456789')");
+ conn.createStatement().execute("UPDATE STATISTICS " + fullTableName);
+ ResultSet rs;
+ String explainPlan;
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT v2 FROM " + fullTableName + " WHERE v2='foo'");
+ explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 3-CHUNK 0 ROWS 0 BYTES PARALLEL 3-WAY FULL SCAN OVER " + fullTableName + "\n" +
+ " SERVER FILTER BY B.V2 = 'foo'\n" +
+ "CLIENT MERGE SORT",
+ explainPlan);
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName);
+ explainPlan = QueryUtil.getExplainPlan(rs);
+ assertEquals(
+ "CLIENT 4-CHUNK 1 ROWS 34 BYTES PARALLEL 3-WAY FULL SCAN OVER " + fullTableName + "\n" +
+ "CLIENT MERGE SORT",
+ explainPlan);
+
+ conn.close();
+ }
+
+ @Test
public void testUpdateStats() throws SQLException, IOException,
InterruptedException {
Connection conn;
@@ -336,7 +387,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
// and forcing the new stats to be pulled over.
int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
- assertEquals(0, rowCount);
+ assertEquals(10, rowCount);
}
List<KeyRange>keyRanges = getAllSplits(conn, tableName);
assertEquals(nRows+1, keyRanges.size());
@@ -356,7 +407,7 @@ public class StatsCollectorIT extends StatsCollectorAbstractIT {
// If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
// and force us to pull over the new stats
int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
- assertEquals(0, rowCount);
+ assertEquals(5, rowCount);
keyRanges = getAllSplits(conn, tableName);
}
assertEquals(nRows/2+1, keyRanges.size());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index fc3edbe..a48de13 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -95,6 +95,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.io.Closeables;
/**
@@ -120,8 +121,9 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final ParallelScanGrouper scanGrouper;
// TODO: too much nesting here - breakup into new classes.
private final List<List<List<Pair<Scan,Future<PeekingResultIterator>>>>> allFutures;
- private long estimatedRows;
- private long estimatedSize;
+ private Long estimatedRows;
+ private Long estimatedSize;
+ private boolean hasGuidePosts;
static final Function<HRegionLocation, KeyRange> TO_KEY_RANGE = new Function<HRegionLocation, KeyRange>() {
@Override
@@ -401,7 +403,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
* be further parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan
* entire regions worth of data to track where to put the guide posts.
*/
- if (!useStats()) { return GuidePostsInfo.EMPTY_GUIDEPOST; }
+ if (!useStats()) { return GuidePostsInfo.NO_GUIDEPOST; }
GuidePostsInfo gps = null;
PTable table = getTable();
@@ -424,7 +426,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
}
}
- if (gps == null) { return GuidePostsInfo.EMPTY_GUIDEPOST; }
+ if (gps == null) { return GuidePostsInfo.NO_GUIDEPOST; }
return gps;
}
@@ -490,6 +492,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
}
GuidePostsInfo gps = getGuidePosts(whereConditions);
+ hasGuidePosts = gps != GuidePostsInfo.NO_GUIDEPOST;
boolean traverseAllRegions = isSalted || isLocalIndex;
if (!traverseAllRegions) {
byte[] scanStartRow = scan.getStartRow();
@@ -528,63 +531,75 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
DataInput input = null;
PrefixByteDecoder decoder = null;
int guideIndex = 0;
- if (gpsSize > 0) {
- stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength());
- input = new DataInputStream(stream);
- decoder = new PrefixByteDecoder(gps.getMaxLength());
- try {
- while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0
- && currentKey.getLength() != 0) {
- guideIndex++;
- }
- } catch (EOFException e) {}
- }
- byte[] currentKeyBytes = currentKey.copyBytes();
-
- // Merge bisect with guideposts for all but the last region
- while (regionIndex <= stopIndex) {
- byte[] currentGuidePostBytes = currentGuidePost.copyBytes();
- byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY;
- if (regionIndex == stopIndex) {
- endKey = stopKey;
- } else {
- endKey = regionBoundaries.get(regionIndex);
- }
- HRegionLocation regionLocation = regionLocations.get(regionIndex);
- if (isLocalIndex) {
- HRegionInfo regionInfo = regionLocation.getRegionInfo();
- endRegionKey = regionInfo.getEndKey();
- keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey);
+ long estimatedRows = 0;
+ long estimatedSize = 0;
+ try {
+ if (gpsSize > 0) {
+ stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength());
+ input = new DataInputStream(stream);
+ decoder = new PrefixByteDecoder(gps.getMaxLength());
+ try {
+ while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0
+ && currentKey.getLength() != 0) {
+ guideIndex++;
+ }
+ } catch (EOFException e) {}
}
- try {
- while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
- Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
- false);
- estimatedRows += gps.getRowCounts().get(guideIndex);
- estimatedSize += gps.getByteCounts().get(guideIndex);
- scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
- currentKeyBytes = currentGuidePost.copyBytes();
- currentGuidePost = PrefixByteCodec.decode(decoder, input);
- currentGuidePostBytes = currentGuidePost.copyBytes();
- guideIndex++;
+ byte[] currentKeyBytes = currentKey.copyBytes();
+
+ // Merge bisect with guideposts for all but the last region
+ while (regionIndex <= stopIndex) {
+ byte[] currentGuidePostBytes = currentGuidePost.copyBytes();
+ byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY;
+ if (regionIndex == stopIndex) {
+ endKey = stopKey;
+ } else {
+ endKey = regionBoundaries.get(regionIndex);
}
- } catch (EOFException e) {}
- Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true);
- if (isLocalIndex) {
- if (newScan != null) {
- newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
- } else if (!scans.isEmpty()) {
- scans.get(scans.size()-1).setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
+ HRegionLocation regionLocation = regionLocations.get(regionIndex);
+ if (isLocalIndex) {
+ HRegionInfo regionInfo = regionLocation.getRegionInfo();
+ endRegionKey = regionInfo.getEndKey();
+ keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey);
}
+ try {
+ while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) {
+ Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset,
+ false);
+ estimatedRows += gps.getRowCounts().get(guideIndex);
+ estimatedSize += gps.getByteCounts().get(guideIndex);
+ scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
+ currentKeyBytes = currentGuidePost.copyBytes();
+ currentGuidePost = PrefixByteCodec.decode(decoder, input);
+ currentGuidePostBytes = currentGuidePost.copyBytes();
+ guideIndex++;
+ }
+ } catch (EOFException e) {}
+ Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true);
+ if (isLocalIndex) {
+ if (newScan != null) {
+ newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
+ } else if (!scans.isEmpty()) {
+ scans.get(scans.size()-1).setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey);
+ }
+ }
+ scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation);
+ currentKeyBytes = endKey;
+ regionIndex++;
}
- scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation);
- currentKeyBytes = endKey;
- regionIndex++;
- }
- if (!scans.isEmpty()) { // Add any remaining scans
- parallelScans.add(scans);
+ if (hasGuidePosts) {
+ this.estimatedRows = estimatedRows;
+ this.estimatedSize = estimatedSize;
+ } else {
+ this.estimatedRows = null;
+ this.estimatedSize = null;
+ }
+ if (!scans.isEmpty()) { // Add any remaining scans
+ parallelScans.add(scans);
+ }
+ } finally {
+ if (stream != null) Closeables.closeQuietly(stream);
}
- PrefixByteCodec.close(stream);
return parallelScans;
}
@@ -857,8 +872,11 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
StringBuilder buf = new StringBuilder();
buf.append("CLIENT ");
if (displayChunkCount) {
+ boolean displayRowCount = context.getConnection().getQueryServices().getProps().getBoolean(
+ QueryServices.EXPLAIN_ROW_COUNT_ATTRIB,
+ QueryServicesOptions.DEFAULT_EXPLAIN_ROW_COUNT);
buf.append(this.splits.size()).append("-CHUNK ");
- if (estimatedRows > 0) {
+ if (displayRowCount && hasGuidePosts) {
buf.append(estimatedRows).append(" ROWS ");
buf.append(estimatedSize).append(" BYTES ");
}
@@ -867,6 +885,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
explain(buf.toString(),planSteps);
}
+ public Long getEstimatedRowCount() {
+ return this.estimatedRows;
+ }
+
+ public Long getEstimatedByteCount() {
+ return this.estimatedSize;
+ }
+
@Override
public String toString() {
return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 63d4e07..471ac73 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -271,7 +271,7 @@ public interface QueryConstants {
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
// PK columns
PHYSICAL_NAME + " VARCHAR NOT NULL," +
- COLUMN_FAMILY + " VARCHAR," +
+ COLUMN_FAMILY + " VARCHAR NOT NULL," +
GUIDE_POST_KEY + " VARBINARY," +
GUIDE_POSTS_WIDTH + " BIGINT," +
LAST_STATS_UPDATE_TIME+ " DATE, "+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 1efcd8c..32b4fc0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -162,6 +162,7 @@ public interface QueryServices extends SQLCloseable {
public static final String SEQUENCE_SALT_BUCKETS_ATTRIB = "phoenix.sequence.saltBuckets";
public static final String COPROCESSOR_PRIORITY_ATTRIB = "phoenix.coprocessor.priority";
public static final String EXPLAIN_CHUNK_COUNT_ATTRIB = "phoenix.explain.displayChunkCount";
+ public static final String EXPLAIN_ROW_COUNT_ATTRIB = "phoenix.explain.displayRowCount";
public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 27c5693..3e0ffe1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -29,6 +29,7 @@ import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB
import static org.apache.phoenix.query.QueryServices.DELAY_FOR_SCHEMA_UPDATE_CHECK;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.EXPLAIN_ROW_COUNT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GLOBAL_METRICS_ENABLED;
@@ -196,6 +197,7 @@ public class QueryServicesOptions {
*/
public static final int DEFAULT_COPROCESSOR_PRIORITY = Coprocessor.PRIORITY_SYSTEM/2 + Coprocessor.PRIORITY_USER/2; // Divide individually to prevent any overflow
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = true;
+ public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = true;
public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
@@ -573,6 +575,11 @@ public class QueryServicesOptions {
return this;
}
+ public QueryServicesOptions setExplainRowCount(boolean showRowCount) {
+ config.setBoolean(EXPLAIN_ROW_COUNT_ATTRIB, showRowCount);
+ return this;
+ }
+
public QueryServicesOptions setAllowOnlineSchemaUpdate(boolean allow) {
config.setBoolean(ALLOW_ONLINE_TABLE_SCHEMA_UPDATE, allow);
return this;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index cb6f5d4..b81d206 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -24,13 +24,11 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
@@ -50,18 +48,15 @@ import com.google.common.collect.Maps;
/**
* A default implementation of the Statistics tracker that helps to collect stats like min key, max key and guideposts.
- * TODO: review timestamps used for stats. We support the user controlling the timestamps, so we should honor that with
- * timestamps for stats as well. The issue is for compaction, though. I don't know of a way for the user to specify any
- * timestamp for that. Perhaps best to use current time across the board for now.
*/
class DefaultStatisticsCollector implements StatisticsCollector {
private static final Logger logger = LoggerFactory.getLogger(DefaultStatisticsCollector.class);
+ private final Map<ImmutableBytesPtr, Pair<Long, GuidePostsInfoBuilder>> guidePostsInfoWriterMap = Maps.newHashMap();
+ private final StatisticsWriter statsWriter;
+ private final Pair<Long, GuidePostsInfoBuilder> cachedGps;
private long guidepostDepth;
private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
- private Map<ImmutableBytesPtr, Pair<Long, GuidePostsInfoBuilder>> guidePostsInfoWriterMap = Maps.newHashMap();
- protected StatisticsWriter statsTable;
- private Pair<Long, GuidePostsInfoBuilder> cachedGps = null;
DefaultStatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family,
byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
@@ -87,12 +82,14 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
// Get the stats table associated with the current table on which the CP is
// triggered
- this.statsTable = StatisticsWriter.newWriter(env, tableName, clientTimeStamp);
+ this.statsWriter = StatisticsWriter.newWriter(env, tableName, clientTimeStamp);
// in a compaction we know the one family ahead of time
if (family != null) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family);
cachedGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
guidePostsInfoWriterMap.put(cfKey, cachedGps);
+ } else {
+ cachedGps = null;
}
}
@@ -103,7 +100,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
@Override
public void close() throws IOException {
- this.statsTable.close();
+ this.statsWriter.close();
}
@Override
@@ -129,9 +126,13 @@ class DefaultStatisticsCollector implements StatisticsCollector {
// Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS
// This will not impact a stats collection of single column family during compaction as
// guidePostsInfoWriterMap cannot be empty in this case.
- if (guidePostsInfoWriterMap.keySet().isEmpty()) {
+ if (cachedGps == null) {
for (Store store : region.getStores()) {
- statsTable.deleteStats(region, this, new ImmutableBytesPtr(store.getFamily().getName()), mutations);
+ ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
+ if (!guidePostsInfoWriterMap.containsKey(cfKey)) {
+ Pair<Long, GuidePostsInfoBuilder> emptyGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
+ guidePostsInfoWriterMap.put(cfKey, emptyGps);
+ }
}
}
for (ImmutableBytesPtr fam : guidePostsInfoWriterMap.keySet()) {
@@ -139,12 +140,12 @@ class DefaultStatisticsCollector implements StatisticsCollector {
if (logger.isDebugEnabled()) {
logger.debug("Deleting the stats for the region " + region.getRegionInfo());
}
- statsTable.deleteStats(region, this, fam, mutations);
+ statsWriter.deleteStats(region, this, fam, mutations);
}
if (logger.isDebugEnabled()) {
logger.debug("Adding new stats for the region " + region.getRegionInfo());
}
- statsTable.addStats(this, fam, mutations);
+ statsWriter.addStats(this, fam, mutations);
}
} catch (IOException e) {
logger.error("Failed to update statistics table!", e);
@@ -153,7 +154,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
private void commitStats(List<Mutation> mutations) throws IOException {
- statsTable.commitStats(mutations);
+ statsWriter.commitStats(mutations);
}
/**
@@ -212,7 +213,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
protected InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, InternalScanner internalScan,
ImmutableBytesPtr family) {
- return new StatisticsScanner(this, statsTable, env, internalScan, family);
+ return new StatisticsScanner(this, statsWriter, env, internalScan, family);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
index b8cc3f1..291cb6d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
@@ -38,7 +38,7 @@ public class GuidePostsInfo {
*/
private int maxLength;
- public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
+ public final static GuidePostsInfo NO_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0);
public int getMaxLength() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
index 2133349..10d59ee 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteEncoder;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
@@ -107,15 +106,10 @@ public class GuidePostsInfoBuilder {
return addGuidePosts(new ImmutableBytesWritable(row), byteCount, rowCount);
}
- private void close() {
- PrefixByteCodec.close(stream);
- }
-
public GuidePostsInfo build() {
this.guidePosts.set(stream.getBuffer(), 0, stream.size());
GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCounts, this.guidePosts, this.rowCounts,
this.maxLength, this.guidePostsCount);
- this.close();
return guidePostsInfo;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
index dacc213..42e7bed 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
@@ -32,6 +32,7 @@ import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteDecoder;
import org.apache.phoenix.util.SizedUtil;
+import com.google.common.io.Closeables;
import com.sun.istack.NotNull;
/**
@@ -89,13 +90,13 @@ public class PTableStatsImpl implements PTableStats {
} catch (EOFException e) { // Ignore as this signifies we're done
} finally {
- PrefixByteCodec.close(stream);
+ Closeables.closeQuietly(stream);
}
buf.setLength(buf.length() - 1);
}
buf.append(")");
} finally {
- PrefixByteCodec.close(stream);
+ Closeables.closeQuietly(stream);
}
}
buf.append("]");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 5b47104..5e03be5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -62,15 +62,19 @@ public class StatisticsUtil {
public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) {
// always starts with the source table
- byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.getLength() + 2];
+ int guidePostLength = guidePostStartKey.getLength();
+ boolean hasGuidePost = guidePostLength > 0;
+ byte[] rowKey = new byte[table.length + fam.getLength() + guidePostLength + (hasGuidePost ? 2 : 1)];
int offset = 0;
System.arraycopy(table, 0, rowKey, offset, table.length);
offset += table.length;
rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength());
- offset += fam.getLength();
- rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
- System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostStartKey.getLength());
+ if (hasGuidePost) {
+ offset += fam.getLength();
+ rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC
+ System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostLength);
+ }
return rowKey;
}
@@ -215,9 +219,12 @@ public class StatisticsUtil {
}
public static byte[] getGuidePostsInfoFromRowKey(byte[] tableNameBytes, byte[] fam, byte[] row) {
- ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- int gpOffset = tableNameBytes.length + 1 + fam.length + 1;
- ptr.set(row, gpOffset, row.length - gpOffset);
- return ByteUtil.copyKeyBytesIfNecessary(ptr);
+ if (row.length > tableNameBytes.length + 1 + fam.length) {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ int gpOffset = tableNameBytes.length + 1 + fam.length + 1;
+ ptr.set(row, gpOffset, row.length - gpOffset);
+ return ByteUtil.copyKeyBytesIfNecessary(ptr);
+ }
+ return ByteUtil.EMPTY_BYTE_ARRAY;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index f8c888d..a727c1c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -49,7 +49,6 @@ import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteDecoder;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.TimeKeeper;
@@ -143,33 +142,46 @@ public class StatisticsWriter implements Closeable {
List<Long> byteCounts = gps.getByteCounts();
List<Long> rowCounts = gps.getRowCounts();
ImmutableBytesWritable keys = gps.getGuidePosts();
- ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength());
- DataInput input = new DataInputStream(stream);
- PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength());
- int guidePostCount = 0;
- try {
- while (true) {
- ImmutableBytesWritable ptr = decoder.decode(input);
- byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr);
- Put put = new Put(prefix);
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
- timeStamp, PLong.INSTANCE.toBytes(byteCounts.get(guidePostCount)));
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
- PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
- PLong.INSTANCE.toBytes(rowCounts.get(guidePostCount)));
- // Add our empty column value so queries behave correctly
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
- ByteUtil.EMPTY_BYTE_ARRAY);
- mutations.add(put);
- guidePostCount++;
- }
- } catch (EOFException e) { // Ignore as this signifies we're done
+ boolean hasGuidePosts = keys.getLength() > 0;
+ if (hasGuidePosts) {
+ int guidePostCount = 0;
+ try (ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength())) {
+ DataInput input = new DataInputStream(stream);
+ PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength());
+ do {
+ ImmutableBytesWritable ptr = decoder.decode(input);
+ addGuidepost(cfKey, mutations, ptr, byteCounts.get(guidePostCount), rowCounts.get(guidePostCount), timeStamp);
+ guidePostCount++;
+ } while (decoder != null);
+ } catch (EOFException e) { // Ignore as this signifies we're done
- } finally {
- PrefixByteCodec.close(stream);
+ }
+ // If we've written guideposts with a guidepost key, then delete the
+ // empty guidepost indicator that may have been written by other
+ // regions.
+ byte[] rowKey = StatisticsUtil.getRowKey(tableName, cfKey, ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY);
+ Delete delete = new Delete(rowKey, timeStamp);
+ mutations.add(delete);
+ } else {
+ addGuidepost(cfKey, mutations, ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY, 0, 0, timeStamp);
}
}
}
+
+ @SuppressWarnings("deprecation")
+ private void addGuidepost(ImmutableBytesPtr cfKey, List<Mutation> mutations, ImmutableBytesWritable ptr, long byteCount, long rowCount, long timeStamp) {
+ byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr);
+ Put put = new Put(prefix);
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
+ timeStamp, PLong.INSTANCE.toBytes(byteCount));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
+ PLong.INSTANCE.toBytes(rowCount));
+ // Add our empty column value so queries behave correctly
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
+ ByteUtil.EMPTY_BYTE_ARRAY);
+ mutations.add(put);
+ }
private static MutationType getMutationType(Mutation m) throws IOException {
if (m instanceof Put) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
index 8c3aa80..2641b6a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java
@@ -18,7 +18,6 @@
package org.apache.phoenix.util;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
@@ -52,15 +51,15 @@ public class PrefixByteCodec {
}
public static int encodeBytes(List<byte[]> listOfBytes, ImmutableBytesWritable ptr) throws IOException {
- TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes));
- DataOutput output = new DataOutputStream(stream);
- PrefixByteEncoder encoder = new PrefixByteEncoder();
- for (byte[] bytes : listOfBytes) {
- encoder.encode(output, bytes, 0, bytes.length);
+ try (TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes))) {
+ DataOutput output = new DataOutputStream(stream);
+ PrefixByteEncoder encoder = new PrefixByteEncoder();
+ for (byte[] bytes : listOfBytes) {
+ encoder.encode(output, bytes, 0, bytes.length);
+ }
+ ptr.set(stream.getBuffer(), 0, stream.size());
+ return encoder.getMaxLength();
}
- close(stream);
- ptr.set(stream.getBuffer(), 0, stream.size());
- return encoder.getMaxLength();
}
public static int calculateSize(List<byte[]> listOfBytes) {
@@ -81,24 +80,4 @@ public class PrefixByteCodec {
throw new RuntimeException(e);
}
}
-
- public static void close(ByteArrayInputStream stream) {
- if (stream != null) {
- try {
- stream.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- public static void close(ByteArrayOutputStream stream) {
- if (stream != null) {
- try {
- stream.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c25f8879/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 29a7001..6ae655c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -54,6 +54,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
public static final boolean DEFAULT_EXPLAIN_CHUNK_COUNT = false; // TODO: update explain plans in test and set to true
+ public static final boolean DEFAULT_EXPLAIN_ROW_COUNT = false; // TODO: update explain plans in test and set to true
public static final String DEFAULT_EXTRA_JDBC_ARGUMENTS = PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
private static final boolean DEFAULT_RUN_UPDATE_STATS_ASYNC = false;
private static final boolean DEFAULT_COMMIT_STATS_ASYNC = false;
@@ -80,6 +81,7 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static QueryServicesOptions getDefaultServicesOptions() {
return withDefaults()
.setExplainChunkCount(DEFAULT_EXPLAIN_CHUNK_COUNT)
+ .setExplainRowCount(DEFAULT_EXPLAIN_ROW_COUNT)
.setSequenceSaltBuckets(DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS)
.setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE)
[09/12] phoenix git commit: PHOENIX-2713 Backward compatibility fails
with CNF exception with 4.7 client and server when upgraded from 4.6
Posted by ma...@apache.org.
PHOENIX-2713 Backward compatibility fails with CNF exception with 4.7 client and server when upgraded from 4.6
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/29ca38b8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/29ca38b8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/29ca38b8
Branch: refs/heads/calcite
Commit: 29ca38b85a5f2455a007404aaa72c104f20a5afe
Parents: a6de19b
Author: James Taylor <jt...@salesforce.com>
Authored: Wed Feb 24 17:44:08 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed Feb 24 17:44:08 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/29ca38b8/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 37ebc78..3327861 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2384,7 +2384,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
metaConnection = updateSystemCatalogTimestamp(metaConnection,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
- logger.warn("Update of SYSTEM.CATALOG complete");
+ logger.info("Update of SYSTEM.CATALOG complete");
clearCache();
}