You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/10/10 08:58:21 UTC
[1/2] git commit: PHOENIX-1309 Ensure Phoenix table is created for
Local index and view index tables to store guideposts against them
Repository: phoenix
Updated Branches:
refs/heads/4.0 78eff8eeb -> 71d6d1a1e
PHOENIX-1309 Ensure Phoenix table is created for Local index and view index tables to store guideposts against them
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b1d19950
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b1d19950
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b1d19950
Branch: refs/heads/4.0
Commit: b1d19950aed22b01504accba903f1247caaccff3
Parents: 78eff8e
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Oct 9 22:08:22 2014 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Oct 9 22:08:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/BaseViewIT.java | 20 ++--
.../apache/phoenix/end2end/SaltedViewIT.java | 2 +-
.../java/org/apache/phoenix/end2end/ViewIT.java | 11 +-
.../UngroupedAggregateRegionObserver.java | 1 +
.../phoenix/iterate/ParallelIterators.java | 107 ++++++++++---------
.../phoenix/query/ConnectionQueryServices.java | 6 ++
.../query/ConnectionQueryServicesImpl.java | 39 ++++++-
.../query/ConnectionlessQueryServicesImpl.java | 14 +++
.../query/DelegateConnectionQueryServices.java | 16 +++
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 7 +-
.../apache/phoenix/schema/MetaDataClient.java | 76 +++++++++++--
.../phoenix/schema/stats/PTableStatsImpl.java | 23 ++++
.../schema/stats/StatisticsCollector.java | 11 +-
.../phoenix/schema/stats/StatisticsWriter.java | 2 +-
.../phoenix/query/QueryServicesTestImpl.java | 2 -
16 files changed, 261 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
index e2c5420..4255e3f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
@@ -27,6 +27,7 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.util.Map;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
@@ -34,18 +35,16 @@ import org.junit.experimental.categories.Category;
import com.google.common.collect.Maps;
-@Category(HBaseManagedTimeTest.class)
-public class BaseViewIT extends BaseHBaseManagedTimeIT {
+@Category(NeedsOwnMiniClusterTest.class)
+public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
@BeforeClass
- @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
public static void doSetup() throws Exception {
Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
- // Don't split intra region so we can more easily know that the n-way parallelization is for the explain plan
- // Must update config before starting server
+ props.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Integer.toString(20));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
-
+
protected void testUpdatableViewWithIndex(Integer saltBuckets, boolean localIndex) throws Exception {
testUpdatableView(saltBuckets);
testUpdatableViewIndex(saltBuckets, localIndex);
@@ -110,6 +109,11 @@ public class BaseViewIT extends BaseHBaseManagedTimeIT {
conn.createStatement().execute("CREATE INDEX i1 on v(k3) include (s)");
}
conn.createStatement().execute("UPSERT INTO v(k2,S,k3) VALUES(120,'foo',50.0)");
+
+// analyzeTable(conn, "v");
+// List<KeyRange> splits = getAllSplits(conn, "i1");
+// assertEquals(4, splits.size());
+
String query = "SELECT k1, k2, k3, s FROM v WHERE k3 = 51.0";
rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -134,6 +138,10 @@ public class BaseViewIT extends BaseHBaseManagedTimeIT {
} else {
conn.createStatement().execute("CREATE INDEX i2 on v(s)");
}
+
+// splits = getAllSplits(conn, "i2");
+// assertEquals(4, splits.size());
+
query = "SELECT k1, k2, s FROM v WHERE s = 'foo'";
rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
index 0db0408..a492d4f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SaltedViewIT.java
@@ -20,7 +20,7 @@ package org.apache.phoenix.end2end;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-@Category(HBaseManagedTimeTest.class)
+@Category(NeedsOwnMiniClusterTest.class)
public class SaltedViewIT extends BaseViewIT {
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
index 20f606d..63beeed 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.end2end;
+import static org.apache.phoenix.util.TestUtil.analyzeTable;
+import static org.apache.phoenix.util.TestUtil.getAllSplits;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -26,14 +28,16 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.List;
import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.ReadOnlyTableException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-@Category(HBaseManagedTimeTest.class)
+@Category(NeedsOwnMiniClusterTest.class)
public class ViewIT extends BaseViewIT {
@Test
@@ -55,6 +59,11 @@ public class ViewIT extends BaseViewIT {
}
conn.commit();
+ analyzeTable(conn, "v");
+
+ List<KeyRange> splits = getAllSplits(conn, "v");
+ assertEquals(4, splits.size());
+
int count = 0;
ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM v");
while (rs.next()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 273ec0e..710409f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -148,6 +148,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
// individual tenant specific tables.
scan.setStartRow(HConstants.EMPTY_START_ROW);
scan.setStopRow(HConstants.EMPTY_END_ROW);
+ scan.setFilter(null);
}
return s;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 7c73918..8ec5215 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -44,7 +44,6 @@ import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.compile.RowProjector;
import org.apache.phoenix.compile.ScanRanges;
import org.apache.phoenix.compile.StatementContext;
-import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.filter.ColumnProjectionFilter;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.job.JobManager.JobCallable;
@@ -54,16 +53,15 @@ import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.ViewType;
-import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.trace.util.Tracing;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.LogUtil;
@@ -92,7 +90,8 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
private static final Logger logger = LoggerFactory.getLogger(ParallelIterators.class);
private final List<List<Scan>> scans;
private final List<KeyRange> splits;
- private final PTable physicalTable;
+ private final PTableStats tableStats;
+ private final byte[] physicalTableName;
private final QueryPlan plan;
private final ParallelIteratorFactory iteratorFactory;
@@ -110,34 +109,37 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
}
};
+ private PTable getTable() {
+ return plan.getTableRef().getTable();
+ }
+
+ private boolean useStats() {
+ Scan scan = context.getScan();
+ boolean isPointLookup = context.getScanRanges().isPointLookup();
+ /*
+ * Don't use guide posts if:
+ * 1) We're doing a point lookup, as HBase is fast enough at those
+ * to not need them to be further parallelized. TODO: pref test to verify
+ * 2) We're collecting stats, as in this case we need to scan entire
+ * regions worth of data to track where to put the guide posts.
+ */
+ if (isPointLookup || ScanUtil.isAnalyzeTable(scan)) {
+ return false;
+ }
+ return true;
+ }
+
public ParallelIterators(QueryPlan plan, Integer perScanLimit, ParallelIteratorFactory iteratorFactory)
throws SQLException {
super(plan.getContext(), plan.getTableRef(), plan.getGroupBy());
this.plan = plan;
StatementContext context = plan.getContext();
TableRef tableRef = plan.getTableRef();
+ PTable table = tableRef.getTable();
FilterableStatement statement = plan.getStatement();
RowProjector projector = plan.getProjector();
- PTable physicalTable = tableRef.getTable();
- String physicalName = tableRef.getTable().getPhysicalName().getString();
- if ((physicalTable.getViewIndexId() == null) && (!physicalName.equals(physicalTable.getName().getString()))) { // tableRef is not for the physical table
- MetaDataClient client = new MetaDataClient(context.getConnection());
- String physicalSchemaName = SchemaUtil.getSchemaNameFromFullName(physicalName);
- String physicalTableName = SchemaUtil.getTableNameFromFullName(physicalName);
- // TODO: this will be an extra RPC to ensure we have the latest guideposts, but is almost always
- // unnecessary. We should instead track when the last time an update cache was done for this
- // for physical table and not do it again until some interval has passed (it's ok to use stale stats).
- MetaDataMutationResult result = client.updateCache(null, /* use global tenant id to get physical table */
- physicalSchemaName, physicalTableName);
- physicalTable = result.getTable();
- if(physicalTable == null) {
- client = new MetaDataClient(context.getConnection());
- physicalTable = client.getConnection().getMetaDataCache()
- .getTable(new PTableKey(null, physicalTableName));
- }
- }
- this.physicalTable = physicalTable;
- PTable table = tableRef.getTable();
+ physicalTableName = table.getPhysicalName().getBytes();
+ tableStats = useStats() ? new MetaDataClient(context.getConnection()).getTableStats(table) : PTableStats.EMPTY_STATS;
Scan scan = context.getScan();
if (projector.isProjectEmptyKeyValue()) {
Map<byte [], NavigableSet<byte []>> familyMap = scan.getFamilyMap();
@@ -301,11 +303,7 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
return guideIndex;
}
- private List<byte[]> getGuidePosts(PTable table) {
- Scan scan = context.getScan();
- boolean isPointLookup = context.getScanRanges().isPointLookup();
- byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(table);
- List<byte[]> gps = Collections.emptyList();
+ private List<byte[]> getGuidePosts() {
/*
* Don't use guide posts if:
* 1) We're doing a point lookup, as HBase is fast enough at those
@@ -313,24 +311,31 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
* 2) We're collecting stats, as in this case we need to scan entire
* regions worth of data to track where to put the guide posts.
*/
- if (!isPointLookup && !ScanUtil.isAnalyzeTable(scan)) {
- if (table.getColumnFamilies().isEmpty()) {
- // For sure we can get the defaultCF from the table
- return table.getGuidePosts();
- }
- try {
- if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) {
- // If default CF is not used in scan, use first CF referenced in scan
- return table.getColumnFamily(scan.getFamilyMap().keySet().iterator().next()).getGuidePosts();
- }
+ if (!useStats()) {
+ return Collections.emptyList();
+ }
+
+ List<byte[]> gps = null;
+ PTable table = getTable();
+ Map<byte[],List<byte[]>> guidePostMap = tableStats.getGuidePosts();
+ byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable());
+ if (table.getColumnFamilies().isEmpty()) {
+ // For sure we can get the defaultCF from the table
+ gps = guidePostMap.get(defaultCF);
+ } else {
+ Scan scan = context.getScan();
+ if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) {
+ // If default CF is not used in scan, use first CF referenced in scan
+ gps = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next());
+ } else {
// Otherwise, favor use of default CF.
- return table.getColumnFamily(defaultCF).getGuidePosts();
- } catch (ColumnFamilyNotFoundException cfne) {
- // Alter table does this
+ gps = guidePostMap.get(defaultCF);
}
}
+ if (gps == null) {
+ return Collections.emptyList();
+ }
return gps;
-
}
private static String toString(List<byte[]> gps) {
@@ -351,14 +356,15 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
if (scan == null) {
return scans;
}
+ PTable table = getTable();
if (!scans.isEmpty()) {
boolean startNewScanList = false;
if (!plan.isRowKeyOrdered()) {
startNewScanList = true;
} else if (crossedRegionBoundary) {
- if (physicalTable.getIndexType() == IndexType.LOCAL) {
+ if (table.getIndexType() == IndexType.LOCAL) {
startNewScanList = true;
- } else if (physicalTable.getBucketNum() != null) {
+ } else if (table.getBucketNum() != null) {
byte[] previousStartKey = scans.get(scans.size()-1).getStartRow();
byte[] currentStartKey = scan.getStartRow();
byte[] prefix = ScanUtil.getPrefix(previousStartKey, SaltingUtil.NUM_SALTING_BYTES);
@@ -382,12 +388,13 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
*/
private List<List<Scan>> getParallelScans(final Scan scan) throws SQLException {
List<HRegionLocation> regionLocations = context.getConnection().getQueryServices()
- .getAllTableRegions(physicalTable.getPhysicalName().getBytes());
+ .getAllTableRegions(physicalTableName);
List<byte[]> regionBoundaries = toBoundaries(regionLocations);
ScanRanges scanRanges = context.getScanRanges();
- boolean isSalted = physicalTable.getBucketNum() != null;
- boolean isLocalIndex = physicalTable.getIndexType() == IndexType.LOCAL;
- List<byte[]> gps = getGuidePosts(physicalTable);
+ PTable table = getTable();
+ boolean isSalted = table.getBucketNum() != null;
+ boolean isLocalIndex = table.getIndexType() == IndexType.LOCAL;
+ List<byte[]> gps = getGuidePosts();
if (logger.isDebugEnabled()) {
logger.debug("Guideposts: " + toString(gps));
}
@@ -489,7 +496,7 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
} catch (StaleRegionBoundaryCacheException e2) { // Catch only to try to recover from region boundary cache being out of date
List<List<Pair<Scan,Future<PeekingResultIterator>>>> newFutures = Lists.newArrayListWithExpectedSize(2);
if (!clearedCache) { // Clear cache once so that we rejigger job based on new boundaries
- services.clearTableRegionCache(physicalTable.getName().getBytes());
+ services.clearTableRegionCache(physicalTableName);
clearedCache = true;
}
// Resubmit just this portion of work again
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index e156555..9d2e194 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -39,6 +39,7 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceKey;
+import org.apache.phoenix.schema.stats.PTableStats;
public interface ConnectionQueryServices extends QueryServices, MetaDataMutated {
@@ -104,4 +105,9 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public String getUserName();
public void incrementTableTimeStamp(final byte[] tenantId, final byte[] schemaName, final byte[] tableName, long clientTS) throws SQLException;
+
+ public PTableStats getTableStats(String physicalName);
+ public void addTableStats(String physicalName, PTableStats tableStats);
+
+ public void clearCache() throws SQLException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 9f30236..40b6bc4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -32,6 +32,7 @@ import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
import javax.annotation.concurrent.GuardedBy;
@@ -121,6 +122,7 @@ import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.Closeables;
import org.apache.phoenix.util.ConfigUtil;
@@ -135,6 +137,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Throwables;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -145,12 +149,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class);
private static final int INITIAL_CHILD_SERVICES_CAPACITY = 100;
private static final int DEFAULT_OUT_OF_ORDER_MUTATIONS_WAIT_TIME_MS = 1000;
+ // Max number of cached table stats for view or shared index physical tables
+ private static final int MAX_TABLE_STATS_CACHE_ENTRIES = 512;
protected final Configuration config;
// Copy of config.getProps(), but read-only to prevent synchronization that we
// don't need.
private final ReadOnlyProps props;
private final String userName;
private final ConcurrentHashMap<ImmutableBytesWritable,ConnectionQueryServices> childServices;
+ private final Cache<String, PTableStats> tableStatsCache;
// Cache the latest meta data here for future connections
// writes guarded by "latestMetaDataLock"
@@ -211,6 +218,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// find the HBase version and use that to determine the KeyValueBuilder that should be used
String hbaseVersion = VersionInfo.getVersion();
this.kvBuilder = KeyValueBuilder.get(hbaseVersion);
+ long halfStatsUpdateFreq = config.getLong(
+ QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2;
+ tableStatsCache = CacheBuilder.newBuilder()
+ .maximumSize(MAX_TABLE_STATS_CACHE_ENTRIES)
+ .expireAfterWrite(halfStatsUpdateFreq, TimeUnit.MILLISECONDS)
+ .build();
}
private void openConnection() throws SQLException {
@@ -309,6 +323,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
} finally {
try {
+ tableStatsCache.invalidateAll();
super.close();
} catch (SQLException e) {
if (sqlE == null) {
@@ -1190,11 +1205,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (dropMetadata) {
dropTables(result.getTableNamesToDelete());
}
+ invalidateTables(result.getTableNamesToDelete());
if (tableType == PTableType.TABLE) {
byte[] physicalTableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData);
ensureViewIndexTableDropped(physicalTableName, timestamp);
ensureLocalIndexTableDropped(physicalTableName, timestamp);
+ tableStatsCache.invalidate(SchemaUtil.getTableName(schemaBytes, tableBytes));
}
break;
default:
@@ -1203,6 +1220,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return result;
}
+ private void invalidateTables(final List<byte[]> tableNamesToDelete) {
+ if (tableNamesToDelete != null) {
+ for ( byte[] tableName : tableNamesToDelete ) {
+ tableStatsCache.invalidate(Bytes.toString(tableName));
+ }
+ }
+ }
+
private void dropTables(final List<byte[]> tableNamesToDelete) throws SQLException {
HBaseAdmin admin = null;
SQLException sqlE = null;
@@ -1405,6 +1430,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (dropMetadata) {
dropTables(result.getTableNamesToDelete());
}
+ invalidateTables(result.getTableNamesToDelete());
break;
default:
break;
@@ -1570,7 +1596,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
* Clears the Phoenix meta data cache on each region server
* @throws SQLException
*/
- protected void clearCache() throws SQLException {
+ @Override
+ public void clearCache() throws SQLException {
try {
SQLException sqlE = null;
HTableInterface htable = this.getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
@@ -1596,6 +1623,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
sqlE = new SQLException(e);
} finally {
try {
+ tableStatsCache.invalidateAll();
htable.close();
} catch (IOException e) {
if (sqlE == null) {
@@ -2055,4 +2083,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void throwConnectionClosedException() {
throw new IllegalStateException("Connection to the cluster is closed");
}
+ @Override
+ public PTableStats getTableStats(String physicalName) {
+ return tableStatsCache.getIfPresent(physicalName);
+ }
+
+ @Override
+ public void addTableStats(String physicalName, PTableStats tableStats) {
+ tableStatsCache.put(physicalName, tableStats);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 37e46b7..5b92439 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -68,6 +68,7 @@ import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
@@ -420,4 +421,17 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
public String getUserName() {
return userName;
}
+
+ @Override
+ public PTableStats getTableStats(String physicalName) {
+ return PTableStats.EMPTY_STATS;
+ }
+
+ @Override
+ public void addTableStats(String physicalName, PTableStats tableStats) {
+ }
+
+ @Override
+ public void clearCache() throws SQLException {
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 86523fd..2bcacc6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -41,6 +41,7 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceKey;
+import org.apache.phoenix.schema.stats.PTableStats;
public class DelegateConnectionQueryServices extends DelegateQueryServices implements ConnectionQueryServices {
@@ -232,4 +233,19 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
throws SQLException {
getDelegate().incrementTableTimeStamp(tenantId, schemaName, tableName, clientTS);
}
+
+ @Override
+ public PTableStats getTableStats(String physicalName) {
+ return getDelegate().getTableStats(physicalName);
+ }
+
+ @Override
+ public void addTableStats(String physicalName, PTableStats tableStats) {
+ getDelegate().addTableStats(physicalName, tableStats);
+ }
+
+ @Override
+ public void clearCache() throws SQLException {
+ getDelegate().clearCache();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index dc92183..0734f19 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -135,6 +135,7 @@ public interface QueryServices extends SQLCloseable {
public static final String STATS_UPDATE_FREQ_MS_ATTRIB = "phoenix.stats.updateFrequency";
public static final String MIN_STATS_UPDATE_FREQ_MS_ATTRIB = "phoenix.stats.minUpdateFrequency";
public static final String STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB = "phoenix.stats.guidepost.width";
+ public static final String STATS_GUIDEPOST_PER_REGION_ATTRIB = "phoenix.stats.guidepost.per.region";
/**
* Get executor service used for parallel scans
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 06ea1d1..edcb597 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -144,11 +144,9 @@ public class QueryServicesOptions {
public static final String DEFAULT_TRACING_FREQ = Tracing.Frequency.NEVER.getKey();
public static final double DEFAULT_TRACING_PROBABILITY_THRESHOLD = 0.05;
- public static final long DEFAULT_STATS_HISTOGRAM_DEPTH_BYTE = 1024 * 1024 * 30;
public static final int DEFAULT_STATS_UPDATE_FREQ_MS = 15 * 60000; // 15min
- public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = DEFAULT_STATS_UPDATE_FREQ_MS/2;
-
-
+ public static final int DEFAULT_GUIDE_POSTS_PER_REGION = 20;
+
public static final boolean DEFAULT_USE_REVERSE_SCAN = true;
private final Configuration config;
@@ -203,7 +201,6 @@ public class QueryServicesOptions {
.setIfUnset(GROUPBY_SPILL_FILES_ATTRIB, DEFAULT_GROUPBY_SPILL_FILES)
.setIfUnset(SEQUENCE_CACHE_SIZE_ATTRIB, DEFAULT_SEQUENCE_CACHE_SIZE)
.setIfUnset(SCAN_RESULT_CHUNK_SIZE, DEFAULT_SCAN_RESULT_CHUNK_SIZE)
- .setIfUnset(STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, DEFAULT_STATS_HISTOGRAM_DEPTH_BYTE);
;
// HBase sets this to 1, so we reset it to something more appropriate.
// Hopefully HBase will change this, because we can't know if a user set
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 70a7de1..9b949b7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -88,6 +88,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
@@ -135,11 +136,14 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
+import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -299,13 +303,18 @@ public class MetaDataClient {
return updateCache(tenantId, schemaName, tableName, false);
}
+ private long getClientTimeStamp() {
+ Long scn = connection.getSCN();
+ long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
+ return clientTimeStamp;
+ }
+
private MetaDataMutationResult updateCache(PName tenantId, String schemaName, String tableName,
boolean alwaysHitServer) throws SQLException { // TODO: pass byte[] herez
- Long scn = connection.getSCN();
+ long clientTimeStamp = getClientTimeStamp();
boolean systemTable = SYSTEM_CATALOG_SCHEMA.equals(schemaName);
// System tables must always have a null tenantId
tenantId = systemTable ? null : tenantId;
- long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
PTable table = null;
String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
long tableTimestamp = HConstants.LATEST_TIMESTAMP;
@@ -481,11 +490,11 @@ public class MetaDataClient {
public MutationState updateStatistics(UpdateStatisticsStatement updateStatisticsStmt)
throws SQLException {
// Check before updating the stats if we have reached the configured time to reupdate the stats once again
- final long msMinBetweenUpdates = connection
- .getQueryServices()
- .getProps()
+ ReadOnlyProps props = connection.getQueryServices().getProps();
+ final long msMinBetweenUpdates = props
.getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
- QueryServicesOptions.DEFAULT_MIN_STATS_UPDATE_FREQ_MS);
+ props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
ColumnResolver resolver = FromCompiler.getResolver(updateStatisticsStmt, connection);
PTable table = resolver.getTables().get(0).getTable();
List<PTable> indexes = table.getIndexes();
@@ -2334,4 +2343,59 @@ public class MetaDataClient {
.build().buildException();
}
}
+
+ public PTableStats getTableStats(PTable table) throws SQLException {
+ boolean isView = table.getType() == PTableType.VIEW;
+ boolean isSharedIndex = table.getViewIndexId() != null;
+ if (!isView && !isSharedIndex) {
+ return table.getTableStats();
+ }
+ String physicalName = table.getPhysicalName().getString();
+ // If we have a VIEW or a local or view INDEX, check our cache rather
+ // than updating the cache for that table to prevent an extra roundtrip.
+ PTableStats tableStats = connection.getQueryServices().getTableStats(physicalName);
+ if (tableStats != null) {
+ return tableStats;
+ }
+ if (isView) {
+ String physicalSchemaName = SchemaUtil.getSchemaNameFromFullName(physicalName);
+ String physicalTableName = SchemaUtil.getTableNameFromFullName(physicalName);
+ MetaDataMutationResult result = updateCache(null, /* use global tenant id to get physical table */
+ physicalSchemaName, physicalTableName);
+ PTable physicalTable = result.getTable();
+ if(physicalTable == null) {
+ // We should be able to find the physical table, as we found the logical one
+ // Might mean the physical table as just deleted.
+ logger.warn("Unable to retrieve physical table " + physicalName + " for table " + table.getName().getString());
+ throw new TableNotFoundException(table.getSchemaName().getString(),table.getTableName().getString());
+ }
+ tableStats = physicalTable.getTableStats();
+ } else {
+ /*
+ * Otherwise, we have a shared view. This case is tricky, because we don't have
+ * table metadata for it, only an HBase table. We do have stats, though, so we'll
+ * query them directly here and cache them so we don't keep querying for them.
+ */
+ HTableInterface statsHTable = connection.getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES);
+ try {
+ long clientTimeStamp = getClientTimeStamp();
+ tableStats = StatisticsUtil.readStatistics(statsHTable, table.getPhysicalName().getBytes(), clientTimeStamp);
+ } catch (IOException e) {
+ logger.warn("Unable to read from stats table", e);
+ // Just cache empty stats. We'll try again after some time anyway.
+ tableStats = PTableStats.EMPTY_STATS;
+ } finally {
+ try {
+ statsHTable.close();
+ } catch (IOException e) {
+ // Log, but continue. We have our stats anyway now.
+ logger.warn("Unable to close stats table", e);
+ }
+ }
+ }
+ // Cache these stats so that we don't keep making a roundrip just to get the stats (as
+ // they don't change very often.
+ connection.getQueryServices().addTableStats(physicalName, tableStats);
+ return tableStats;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
index e7114fe..02ecb20 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.schema.stats;
import java.util.List;
+import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -43,4 +44,26 @@ public class PTableStatsImpl implements PTableStats {
public SortedMap<byte[], List<byte[]>> getGuidePosts() {
return guidePosts;
}
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("PTableStats [");
+ for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) {
+ buf.append(Bytes.toStringBinary(entry.getKey()));
+ buf.append(":(");
+ List<byte[]> keys = entry.getValue();
+ if (!keys.isEmpty()) {
+ for (byte[] key : keys) {
+ buf.append(Bytes.toStringBinary(key));
+ buf.append(",");
+ }
+ buf.setLength(buf.length()-1);
+ }
+ buf.append(")");
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 09d5917..49e6b39 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -73,13 +73,16 @@ public class StatisticsCollector {
private static final Log LOG = LogFactory.getLog(StatisticsCollector.class);
public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException {
+ Configuration config = env.getConfiguration();
+ HTableInterface statsHTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES));
guidepostDepth =
- env.getConfiguration().getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_HISTOGRAM_DEPTH_BYTE);
+ config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+ statsHTable.getTableDescriptor().getMaxFileSize() /
+ config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_GUIDE_POSTS_PER_REGION));
// Get the stats table associated with the current table on which the CP is
// triggered
- HTableInterface statsHTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES));
- this.statsTable = StatisticsWriter.getStatisticsTable(statsHTable, tableName, clientTimeStamp);
+ this.statsTable = StatisticsWriter.newWriter(statsHTable, tableName, clientTimeStamp);
}
public void close() throws IOException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index 8e82a88..ddfc2d6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -56,7 +56,7 @@ public class StatisticsWriter implements Closeable {
* @throws IOException
* if the table cannot be created due to an underlying HTable creation error
*/
- public static StatisticsWriter getStatisticsTable(HTableInterface hTable, String tableName, long clientTimeStamp) throws IOException {
+ public static StatisticsWriter newWriter(HTableInterface hTable, String tableName, long clientTimeStamp) throws IOException {
if (clientTimeStamp == HConstants.LATEST_TIMESTAMP) {
clientTimeStamp = TimeKeeper.SYSTEM.getCurrentTime();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b1d19950/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index a7d34da..b7a2df9 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -51,7 +51,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static final String DEFAULT_WAL_EDIT_CODEC = IndexedWALEditCodec.class.getName();
public static final long DEFAULT_MAX_SERVER_METADATA_CACHE_SIZE = 1024L*1024L*4L; // 4 Mb
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*2L; // 2 Mb
- public static final long DEFAULT_STATS_HISTOGRAM_DEPTH_BYTES = 2000;
public static final int DEFAULT_MIN_STATS_UPDATE_FREQ_MS = 0;
@@ -62,7 +61,6 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
private static QueryServicesOptions getDefaultServicesOptions() {
return withDefaults()
.setMinStatsUpdateFrequencyMs(DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
- .setStatsHistogramDepthBytes(DEFAULT_STATS_HISTOGRAM_DEPTH_BYTES)
.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE)
.setQueueSize(DEFAULT_QUEUE_SIZE)
.setMaxMemoryPerc(DEFAULT_MAX_MEMORY_PERC)
[2/2] git commit: PHOENIX-1309 Remove getGuidePosts() methods and
access through PTableStats instead
Posted by ja...@apache.org.
PHOENIX-1309 Remove getGuidePosts() methods and access through PTableStats instead
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/71d6d1a1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/71d6d1a1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/71d6d1a1
Branch: refs/heads/4.0
Commit: 71d6d1a1e657c45b525eec10867bc04c686505bf
Parents: b1d1995
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Oct 10 00:02:06 2014 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Oct 10 00:02:06 2014 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/QueryIT.java | 20 +--
.../coprocessor/generated/PTableProtos.java | 122 ++++---------------
.../query/ConnectionQueryServicesImpl.java | 1 +
.../apache/phoenix/schema/PColumnFamily.java | 3 -
.../phoenix/schema/PColumnFamilyImpl.java | 19 ---
.../java/org/apache/phoenix/schema/PTable.java | 7 --
.../org/apache/phoenix/schema/PTableImpl.java | 53 ++------
.../phoenix/schema/stats/PTableStats.java | 7 ++
.../phoenix/schema/stats/PTableStatsImpl.java | 18 +++
.../phoenix/schema/stats/StatisticsScanner.java | 2 -
.../java/org/apache/phoenix/util/SizedUtil.java | 5 +
phoenix-protocol/src/main/PTable.proto | 2 +-
12 files changed, 79 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
index 07cb624..083c220 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
@@ -204,7 +204,7 @@ public class QueryIT extends BaseQueryIT {
}
private void testNoStringValue(String value) throws Exception {
- String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 1);
+ String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 10);
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection upsertConn = DriverManager.getConnection(url, props);
upsertConn.setAutoCommit(true); // Test auto commit
@@ -215,13 +215,15 @@ public class QueryIT extends BaseQueryIT {
stmt.setString(2, ROW5);
stmt.setString(3, value);
stmt.execute(); // should commit too
+ upsertConn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20));
Connection conn1 = DriverManager.getConnection(getUrl(), props);
analyzeTable(conn1, "ATABLE");
conn1.close();
- upsertConn.close();
String query = "SELECT a_string, b_string FROM aTable WHERE organization_id=? and a_integer = 5";
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
Connection conn = DriverManager.getConnection(getUrl(), props);
try {
PreparedStatement statement = conn.prepareStatement(query);
@@ -813,15 +815,15 @@ public class QueryIT extends BaseQueryIT {
public void testSumOverNullIntegerColumn() throws Exception {
String query = "SELECT sum(a_integer) FROM aTable a";
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20));
Connection conn = DriverManager.getConnection(getUrl(), props);
conn.setAutoCommit(true);
conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) VALUES('" + getOrganizationId() + "','" + ROW3 + "',NULL)");
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 6));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
Connection conn1 = DriverManager.getConnection(getUrl(), props);
analyzeTable(conn1, "ATABLE");
conn1.close();
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 5));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 50));
conn = DriverManager.getConnection(getUrl(), props);
try {
PreparedStatement statement = conn.prepareStatement(query);
@@ -832,15 +834,15 @@ public class QueryIT extends BaseQueryIT {
} finally {
conn.close();
}
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 7));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 70));
conn = DriverManager.getConnection(getUrl(), props);
conn.setAutoCommit(true);
conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) SELECT organization_id, entity_id, null FROM atable");
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 6));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 60));
conn1 = DriverManager.getConnection(getUrl(), props);
analyzeTable(conn1, "ATABLE");
conn1.close();
- props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 9));
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 90));
conn = DriverManager.getConnection(getUrl(), props);
try {
PreparedStatement statement = conn.prepareStatement(query);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index ef0ece2..866870f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@ -1578,20 +1578,15 @@ public final class PTableProtos {
public interface PTableStatsOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // required string key = 1;
+ // required bytes key = 1;
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
boolean hasKey();
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
- java.lang.String getKey();
- /**
- * <code>required string key = 1;</code>
- */
- com.google.protobuf.ByteString
- getKeyBytes();
+ com.google.protobuf.ByteString getKey();
// repeated bytes values = 2;
/**
@@ -1714,47 +1709,20 @@ public final class PTableProtos {
}
private int bitField0_;
- // required string key = 1;
+ // required bytes key = 1;
public static final int KEY_FIELD_NUMBER = 1;
- private java.lang.Object key_;
+ private com.google.protobuf.ByteString key_;
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
public boolean hasKey() {
return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
- public java.lang.String getKey() {
- java.lang.Object ref = key_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- key_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string key = 1;</code>
- */
- public com.google.protobuf.ByteString
- getKeyBytes() {
- java.lang.Object ref = key_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- key_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
+ public com.google.protobuf.ByteString getKey() {
+ return key_;
}
// repeated bytes values = 2;
@@ -1781,7 +1749,7 @@ public final class PTableProtos {
}
private void initFields() {
- key_ = "";
+ key_ = com.google.protobuf.ByteString.EMPTY;
values_ = java.util.Collections.emptyList();
}
private byte memoizedIsInitialized = -1;
@@ -1801,7 +1769,7 @@ public final class PTableProtos {
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getKeyBytes());
+ output.writeBytes(1, key_);
}
for (int i = 0; i < values_.size(); i++) {
output.writeBytes(2, values_.get(i));
@@ -1817,7 +1785,7 @@ public final class PTableProtos {
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getKeyBytes());
+ .computeBytesSize(1, key_);
}
{
int dataSize = 0;
@@ -1988,7 +1956,7 @@ public final class PTableProtos {
public Builder clear() {
super.clear();
- key_ = "";
+ key_ = com.google.protobuf.ByteString.EMPTY;
bitField0_ = (bitField0_ & ~0x00000001);
values_ = java.util.Collections.emptyList();
bitField0_ = (bitField0_ & ~0x00000002);
@@ -2046,9 +2014,7 @@ public final class PTableProtos {
public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats other) {
if (other == org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats.getDefaultInstance()) return this;
if (other.hasKey()) {
- bitField0_ |= 0x00000001;
- key_ = other.key_;
- onChanged();
+ setKey(other.getKey());
}
if (!other.values_.isEmpty()) {
if (values_.isEmpty()) {
@@ -2091,49 +2057,24 @@ public final class PTableProtos {
}
private int bitField0_;
- // required string key = 1;
- private java.lang.Object key_ = "";
+ // required bytes key = 1;
+ private com.google.protobuf.ByteString key_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
public boolean hasKey() {
return ((bitField0_ & 0x00000001) == 0x00000001);
}
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
- public java.lang.String getKey() {
- java.lang.Object ref = key_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- key_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>required string key = 1;</code>
- */
- public com.google.protobuf.ByteString
- getKeyBytes() {
- java.lang.Object ref = key_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- key_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
+ public com.google.protobuf.ByteString getKey() {
+ return key_;
}
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
- public Builder setKey(
- java.lang.String value) {
+ public Builder setKey(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
@@ -2143,7 +2084,7 @@ public final class PTableProtos {
return this;
}
/**
- * <code>required string key = 1;</code>
+ * <code>required bytes key = 1;</code>
*/
public Builder clearKey() {
bitField0_ = (bitField0_ & ~0x00000001);
@@ -2151,19 +2092,6 @@ public final class PTableProtos {
onChanged();
return this;
}
- /**
- * <code>required string key = 1;</code>
- */
- public Builder setKeyBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- key_ = value;
- onChanged();
- return this;
- }
// repeated bytes values = 2;
private java.util.List<com.google.protobuf.ByteString> values_ = java.util.Collections.emptyList();
@@ -5721,7 +5649,7 @@ public final class PTableProtos {
"le\030\005 \001(\005\022\020\n\010nullable\030\006 \002(\010\022\020\n\010position\030\007" +
" \002(\005\022\021\n\tsortOrder\030\010 \002(\005\022\021\n\tarraySize\030\t \001" +
"(\005\022\024\n\014viewConstant\030\n \001(\014\022\026\n\016viewReferenc" +
- "ed\030\013 \001(\010\"*\n\013PTableStats\022\013\n\003key\030\001 \002(\t\022\016\n\006" +
+ "ed\030\013 \001(\010\"*\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006" +
"values\030\002 \003(\014\"\212\004\n\006PTable\022\027\n\017schemaNameByt" +
"es\030\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttabl" +
"eType\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004",
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 40b6bc4..9223b0b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1901,6 +1901,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
sqlE = new SQLException(e);
} finally {
try {
+ if (tenantId.length == 0) tableStatsCache.invalidate(SchemaUtil.getTableName(schemaName, tableName));
htable.close();
} catch (IOException e) {
if (sqlE == null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java
index 01c236f..24da14d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamily.java
@@ -18,7 +18,6 @@
package org.apache.phoenix.schema;
import java.util.Collection;
-import java.util.List;
/**
*
@@ -52,6 +51,4 @@ public interface PColumnFamily {
PColumn getColumn(String name) throws ColumnNotFoundException;
int getEstimatedSize();
-
- List<byte[]> getGuidePosts();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java
index 9841233..2e29656 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PColumnFamilyImpl.java
@@ -17,7 +17,6 @@
*/
package org.apache.phoenix.schema;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -35,7 +34,6 @@ public class PColumnFamilyImpl implements PColumnFamily {
private final Map<String, PColumn> columnByString;
private final Map<byte[], PColumn> columnByBytes;
private final int estimatedSize;
- private List<byte[]> guidePosts = Collections.emptyList();
@Override
public int getEstimatedSize() {
@@ -43,22 +41,10 @@ public class PColumnFamilyImpl implements PColumnFamily {
}
public PColumnFamilyImpl(PName name, List<PColumn> columns) {
- this(name, columns, null);
- }
-
- public PColumnFamilyImpl(PName name, List<PColumn> columns, List<byte[]> guidePosts) {
Preconditions.checkNotNull(name);
// Include guidePosts also in estimating the size
long estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.POINTER_SIZE * 5 + SizedUtil.INT_SIZE + name.getEstimatedSize() +
SizedUtil.sizeOfMap(columns.size()) * 2 + SizedUtil.sizeOfArrayList(columns.size());
- if(guidePosts != null) {
- int guidePostsSize = guidePosts.size();
- estimatedSize += SizedUtil.sizeOfArrayList(guidePostsSize);
- for(byte[] gps : guidePosts) {
- estimatedSize += gps.length;
- }
- this.guidePosts = guidePosts;
- }
this.name = name;
this.columns = ImmutableList.copyOf(columns);
ImmutableMap.Builder<String, PColumn> columnByStringBuilder = ImmutableMap.builder();
@@ -100,9 +86,4 @@ public class PColumnFamilyImpl implements PColumnFamily {
}
return column;
}
-
- @Override
- public List<byte[]> getGuidePosts() {
- return guidePosts;
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 3ea08e1..4193200 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -252,13 +252,6 @@ public interface PTable {
*/
int newKey(ImmutableBytesWritable key, byte[][] values);
- /**
- * Return the statistics table associated with this PTable. A list of
- * guide posts are return
- * @return the statistics table.
- */
- List<byte[]> getGuidePosts();
-
RowKeySchema getRowKeySchema();
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index d883259..fdafc59 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -115,7 +115,6 @@ public class PTableImpl implements PTable {
private Short viewIndexId;
private int estimatedSize;
private IndexType indexType;
- private List<byte[]> guidePosts = Collections.emptyList();
private PTableStats tableStats = PTableStats.EMPTY_STATS;
public PTableImpl() {
@@ -357,24 +356,13 @@ public class PTableImpl implements PTable {
estimatedSize += rowKeySchema.getEstimatedSize();
Iterator<Map.Entry<PName,List<PColumn>>> iterator = familyMap.entrySet().iterator();
PColumnFamily[] families = new PColumnFamily[familyMap.size()];
- if (families.length == 0) {
- byte[] defaultFamilyNameBytes = (defaultFamilyName == null) ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : defaultFamilyName.getBytes();
- List<byte[]> guidePosts = stats.getGuidePosts().get(defaultFamilyNameBytes);
- if (guidePosts != null) {
- this.guidePosts = guidePosts;
- estimatedSize += SizedUtil.sizeOfArrayList(guidePosts.size());
- for (byte[] gps : guidePosts) {
- estimatedSize += gps.length;
- }
- }
- }
+ estimatedSize += this.tableStats.getEstimatedSize();
ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder();
ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap
.orderedBy(Bytes.BYTES_COMPARATOR);
for (int i = 0; i < families.length; i++) {
Map.Entry<PName,List<PColumn>> entry = iterator.next();
- List<byte[]> famGuidePosts = stats.getGuidePosts().get(entry.getKey().getBytes());
- PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue(), famGuidePosts);
+ PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue());
families[i] = family;
familyByString.put(family.getName().getString(), family);
familyByBytes.put(family.getName().getBytes(), family);
@@ -726,11 +714,6 @@ public class PTableImpl implements PTable {
}
@Override
- public List<byte[]> getGuidePosts() {
- return guidePosts;
- }
-
- @Override
public PColumn getPKColumn(String name) throws ColumnNotFoundException {
List<PColumn> columns = columnsByName.get(name);
int size = columns.size();
@@ -892,8 +875,10 @@ public class PTableImpl implements PTable {
for (int j = 0; j < pTableStatsProto.getValuesCount(); j++) {
value.add(pTableStatsProto.getValues(j).toByteArray());
}
- tableGuidePosts.put(pTableStatsProto.getKeyBytes().toByteArray(), value);
+ tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), value);
}
+ PTableStats stats = new PTableStatsImpl(tableGuidePosts);
+
PName dataTableName = null;
if (table.hasDataTableNameBytes()) {
dataTableName = PNameFactory.newName(table.getDataTableNameBytes().toByteArray());
@@ -920,7 +905,6 @@ public class PTableImpl implements PTable {
}
}
- PTableStats stats = new PTableStatsImpl(tableGuidePosts);
try {
PTableImpl result = new PTableImpl();
result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName,
@@ -978,28 +962,13 @@ public class PTableImpl implements PTable {
}
builder.setIsImmutableRows(table.isImmutableRows());
- // build stats for the table
- if (table.getColumnFamilies().isEmpty() && !table.getGuidePosts().isEmpty()) {
- List<byte[]> stats = table.getGuidePosts();
- if (stats != null) {
- PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder();
- statsBuilder.setKey(Bytes.toString(SchemaUtil.getEmptyColumnFamily(table)));
- for (byte[] stat : stats) {
- statsBuilder.addValues(HBaseZeroCopyByteString.wrap(stat));
- }
- builder.addGuidePosts(statsBuilder.build());
+ for (Map.Entry<byte[], List<byte[]>> entry : table.getTableStats().getGuidePosts().entrySet()) {
+ PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder();
+ statsBuilder.setKey(HBaseZeroCopyByteString.wrap(entry.getKey()));
+ for (byte[] stat : entry.getValue()) {
+ statsBuilder.addValues(HBaseZeroCopyByteString.wrap(stat));
}
- } else {
- for (PColumnFamily fam : table.getColumnFamilies()) {
- PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder();
- if (fam.getGuidePosts() != null) {
- statsBuilder.setKey(fam.getName().getString());
- for (byte[] stat : fam.getGuidePosts()) {
- statsBuilder.addValues(HBaseZeroCopyByteString.wrap(stat));
- }
- builder.addGuidePosts(statsBuilder.build());
- }
- }
+ builder.addGuidePosts(statsBuilder.build());
}
if (table.getParentName() != null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
index 40309e4..0782a2b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
@@ -33,6 +33,11 @@ public interface PTableStats {
public SortedMap<byte[], List<byte[]>> getGuidePosts() {
return ImmutableSortedMap.of();
}
+
+ @Override
+ public int getEstimatedSize() {
+ return 0;
+ }
};
/**
@@ -41,4 +46,6 @@ public interface PTableStats {
* @return
*/
SortedMap<byte[], List<byte[]>> getGuidePosts();
+
+ int getEstimatedSize();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
index 02ecb20..5d7d2ac 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
@@ -23,6 +23,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.util.SizedUtil;
import com.sun.istack.NotNull;
@@ -31,6 +32,7 @@ import com.sun.istack.NotNull;
*/
public class PTableStatsImpl implements PTableStats {
private final SortedMap<byte[], List<byte[]>> guidePosts;
+ private final int estimatedSize;
public PTableStatsImpl() {
this(new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR));
@@ -38,6 +40,17 @@ public class PTableStatsImpl implements PTableStats {
public PTableStatsImpl(@NotNull SortedMap<byte[], List<byte[]>> guidePosts) {
this.guidePosts = guidePosts;
+ int estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.INT_SIZE + SizedUtil.sizeOfTreeMap(guidePosts.size());
+ for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) {
+ byte[] cf = entry.getKey();
+ estimatedSize += SizedUtil.ARRAY_SIZE + cf.length;
+ List<byte[]> keys = entry.getValue();
+ estimatedSize += SizedUtil.sizeOfArrayList(keys.size());
+ for (byte[] key : keys) {
+ estimatedSize += SizedUtil.ARRAY_SIZE + key.length;
+ }
+ }
+ this.estimatedSize = estimatedSize;
}
@Override
@@ -65,5 +78,10 @@ public class PTableStatsImpl implements PTableStats {
buf.append("]");
return buf.toString();
}
+
+ @Override
+ public int getEstimatedSize() {
+ return estimatedSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
index 598f0d2..60b9601 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.util.TimeKeeper;
/**
* The scanner that does the scanning to collect the stats during major compaction.{@link StatisticsCollector}
@@ -81,7 +80,6 @@ public class StatisticsScanner implements InternalScanner {
// update the statistics table
// Just verify if this if fine
ArrayList<Mutation> mutations = new ArrayList<Mutation>();
- long currentTime = TimeKeeper.SYSTEM.getCurrentTime();
if (LOG.isDebugEnabled()) {
LOG.debug("Deleting the stats for the region " + region.getRegionNameAsString()
+ " as part of major compaction");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
index c49b0e7..f82c1b8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SizedUtil.java
@@ -38,6 +38,7 @@ public class SizedUtil {
public static final int INT_SIZE = 4;
public static final int LONG_SIZE = 8;
+ public static final int TREE_MAP_SIZE = OBJECT_SIZE + INT_SIZE * 2 + POINTER_SIZE * 2;
public static final int MAP_ENTRY_SIZE = OBJECT_SIZE + 3 * POINTER_SIZE + INT_SIZE;
public static final int IMMUTABLE_BYTES_WRITABLE_SIZE = OBJECT_SIZE + INT_SIZE * 2 + ARRAY_SIZE;
public static final int IMMUTABLE_BYTES_PTR_SIZE = IMMUTABLE_BYTES_WRITABLE_SIZE + INT_SIZE;// Extra is an int field which caches hashcode.
@@ -52,6 +53,10 @@ public class SizedUtil {
private SizedUtil() {
}
+ public static int sizeOfTreeMap(int size) {
+ return TREE_MAP_SIZE + (OBJECT_SIZE + INT_SIZE + POINTER_SIZE * 5) * size;
+ }
+
public static int sizeOfArrayList(int capacity) {
return SizedUtil.OBJECT_SIZE + SizedUtil.POINTER_SIZE + SizedUtil.INT_SIZE + SizedUtil.ARRAY_SIZE + SizedUtil.POINTER_SIZE * capacity;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/71d6d1a1/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto
index 3b5f5cf..0edc046 100644
--- a/phoenix-protocol/src/main/PTable.proto
+++ b/phoenix-protocol/src/main/PTable.proto
@@ -47,7 +47,7 @@ message PColumn {
}
message PTableStats {
- required string key = 1;
+ required bytes key = 1;
repeated bytes values = 2;
}