You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2016/02/24 22:13:48 UTC
[30/50] [abbrv] phoenix git commit: PHOENIX-2683 store rowCount and
byteCount at guidePost level(Ankit Singhal)
PHOENIX-2683 store rowCount and byteCount at guidePost level(Ankit Singhal)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d2fcd21d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d2fcd21d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d2fcd21d
Branch: refs/heads/calcite
Commit: d2fcd21d4f55c17585adbbc41f72027323a8a870
Parents: e797b36
Author: Ankit Singhal <an...@gmail.com>
Authored: Tue Feb 16 22:03:58 2016 +0530
Committer: Ankit Singhal <an...@gmail.com>
Committed: Tue Feb 16 22:03:58 2016 +0530
----------------------------------------------------------------------
.../StatsCollectorWithSplitsAndMultiCFIT.java | 66 +++-
.../generated/PGuidePostsProtos.java | 336 ++++++++++++++++++-
.../org/apache/phoenix/execute/ScanPlan.java | 5 +-
.../org/apache/phoenix/schema/PTableImpl.java | 25 +-
.../phoenix/schema/stats/GuidePostsInfo.java | 69 ++--
.../schema/stats/GuidePostsInfoBuilder.java | 61 ++--
.../schema/stats/StatisticsCollector.java | 21 +-
.../phoenix/schema/stats/StatisticsWriter.java | 18 +-
phoenix-protocol/src/main/PGuidePosts.proto | 2 +
9 files changed, 502 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
index dfe8b60..13cd54c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
@@ -26,9 +26,11 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Random;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.util.Bytes;
@@ -102,10 +104,10 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
TestUtil.analyzeTable(conn, STATS_TEST_TABLE_NAME_NEW);
String query = "UPDATE STATISTICS " + STATS_TEST_TABLE_NAME_NEW + " SET \""
- + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB + "\"=" + Long.toString(2000);
+ + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB + "\"=" + Long.toString(250);
conn.createStatement().execute(query);
keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME_NEW);
- assertEquals(6, keyRanges.size());
+ assertEquals(26, keyRanges.size());
rs = conn.createStatement().executeQuery(
"SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH),COUNT(*) from SYSTEM.STATS where PHYSICAL_NAME = '"
@@ -115,27 +117,71 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
assertTrue(rs.next());
assertEquals("A", rs.getString(1));
assertEquals(25, rs.getInt(2));
- assertEquals(11040, rs.getInt(3));
- assertEquals(5, rs.getInt(4));
+ assertEquals(12420, rs.getInt(3));
+ assertEquals(25, rs.getInt(4));
assertTrue(rs.next());
assertEquals("B", rs.getString(1));
assertEquals(20, rs.getInt(2));
- assertEquals(4432, rs.getInt(3));
- assertEquals(2, rs.getInt(4));
+ assertEquals(5540, rs.getInt(3));
+ assertEquals(20, rs.getInt(4));
assertTrue(rs.next());
assertEquals("C", rs.getString(1));
assertEquals(25, rs.getInt(2));
- assertEquals(6652, rs.getInt(3));
- assertEquals(3, rs.getInt(4));
+ assertEquals(6930, rs.getInt(3));
+ assertEquals(25, rs.getInt(4));
assertTrue(rs.next());
assertEquals("D", rs.getString(1));
assertEquals(25, rs.getInt(2));
- assertEquals(6652, rs.getInt(3));
- assertEquals(3, rs.getInt(4));
+ assertEquals(6930, rs.getInt(3));
+ assertEquals(25, rs.getInt(4));
}
+ @Test
+ public void testRowCountAndByteCounts() throws SQLException {
+ Connection conn;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = "T";
+ String ddl = "CREATE TABLE " + tableName + " (t_id VARCHAR NOT NULL,\n" + "k1 INTEGER NOT NULL,\n"
+ + "k2 INTEGER NOT NULL,\n" + "C3.k3 INTEGER,\n" + "C2.v1 VARCHAR,\n"
+ + "CONSTRAINT pk PRIMARY KEY (t_id, k1, k2)) split on ('e','j','o')";
+ conn.createStatement().execute(ddl);
+ String[] strings = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r",
+ "s", "t", "u", "v", "w", "x", "y", "z" };
+ for (int i = 0; i < 26; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i + ","
+ + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
+ }
+ conn.commit();
+ ResultSet rs;
+ String query = "UPDATE STATISTICS " + tableName + " SET \"" + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB
+ + "\"=" + Long.toString(20);
+ conn.createStatement().execute(query);
+ Random r = new Random();
+ int count = 0;
+ while (count < 4) {
+ int startIndex = r.nextInt(strings.length);
+ int endIndex = r.nextInt(strings.length - startIndex) + startIndex;
+ long rows = endIndex - startIndex;
+ long c2Bytes = rows * 35;
+ System.out.println(rows + ":" + startIndex + ":" + endIndex);
+ rs = conn.createStatement().executeQuery(
+ "SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH) from SYSTEM.STATS where PHYSICAL_NAME = '"
+ + tableName + "' AND GUIDE_POST_KEY>= cast('" + strings[startIndex]
+ + "' as varbinary) AND GUIDE_POST_KEY<cast('" + strings[endIndex]
+ + "' as varbinary) and COLUMN_FAMILY='C2' group by COLUMN_FAMILY");
+ if (startIndex < endIndex) {
+ assertTrue(rs.next());
+ assertEquals("C2", rs.getString(1));
+ assertEquals(rows, rs.getLong(2));
+ assertEquals(c2Bytes, rs.getLong(3));
+ count++;
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java
index d9dcb56..5026307 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java
@@ -74,6 +74,34 @@ public final class PGuidePostsProtos {
* <code>optional bytes encodedGuidePosts = 6;</code>
*/
com.google.protobuf.ByteString getEncodedGuidePosts();
+
+ // repeated int64 byteCounts = 7;
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ java.util.List<java.lang.Long> getByteCountsList();
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ int getByteCountsCount();
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ long getByteCounts(int index);
+
+ // repeated int64 rowCounts = 8;
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ java.util.List<java.lang.Long> getRowCountsList();
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ int getRowCountsCount();
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ long getRowCounts(int index);
}
/**
* Protobuf type {@code PGuidePosts}
@@ -159,6 +187,48 @@ public final class PGuidePostsProtos {
encodedGuidePosts_ = input.readBytes();
break;
}
+ case 56: {
+ if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
+ byteCounts_ = new java.util.ArrayList<java.lang.Long>();
+ mutable_bitField0_ |= 0x00000040;
+ }
+ byteCounts_.add(input.readInt64());
+ break;
+ }
+ case 58: {
+ int length = input.readRawVarint32();
+ int limit = input.pushLimit(length);
+ if (!((mutable_bitField0_ & 0x00000040) == 0x00000040) && input.getBytesUntilLimit() > 0) {
+ byteCounts_ = new java.util.ArrayList<java.lang.Long>();
+ mutable_bitField0_ |= 0x00000040;
+ }
+ while (input.getBytesUntilLimit() > 0) {
+ byteCounts_.add(input.readInt64());
+ }
+ input.popLimit(limit);
+ break;
+ }
+ case 64: {
+ if (!((mutable_bitField0_ & 0x00000080) == 0x00000080)) {
+ rowCounts_ = new java.util.ArrayList<java.lang.Long>();
+ mutable_bitField0_ |= 0x00000080;
+ }
+ rowCounts_.add(input.readInt64());
+ break;
+ }
+ case 66: {
+ int length = input.readRawVarint32();
+ int limit = input.pushLimit(length);
+ if (!((mutable_bitField0_ & 0x00000080) == 0x00000080) && input.getBytesUntilLimit() > 0) {
+ rowCounts_ = new java.util.ArrayList<java.lang.Long>();
+ mutable_bitField0_ |= 0x00000080;
+ }
+ while (input.getBytesUntilLimit() > 0) {
+ rowCounts_.add(input.readInt64());
+ }
+ input.popLimit(limit);
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -170,6 +240,12 @@ public final class PGuidePostsProtos {
if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_);
}
+ if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
+ byteCounts_ = java.util.Collections.unmodifiableList(byteCounts_);
+ }
+ if (((mutable_bitField0_ & 0x00000080) == 0x00000080)) {
+ rowCounts_ = java.util.Collections.unmodifiableList(rowCounts_);
+ }
this.unknownFields = unknownFields.build();
makeExtensionsImmutable();
}
@@ -305,6 +381,52 @@ public final class PGuidePostsProtos {
return encodedGuidePosts_;
}
+ // repeated int64 byteCounts = 7;
+ public static final int BYTECOUNTS_FIELD_NUMBER = 7;
+ private java.util.List<java.lang.Long> byteCounts_;
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public java.util.List<java.lang.Long>
+ getByteCountsList() {
+ return byteCounts_;
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public int getByteCountsCount() {
+ return byteCounts_.size();
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public long getByteCounts(int index) {
+ return byteCounts_.get(index);
+ }
+
+ // repeated int64 rowCounts = 8;
+ public static final int ROWCOUNTS_FIELD_NUMBER = 8;
+ private java.util.List<java.lang.Long> rowCounts_;
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public java.util.List<java.lang.Long>
+ getRowCountsList() {
+ return rowCounts_;
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public int getRowCountsCount() {
+ return rowCounts_.size();
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public long getRowCounts(int index) {
+ return rowCounts_.get(index);
+ }
+
private void initFields() {
guidePosts_ = java.util.Collections.emptyList();
byteCount_ = 0L;
@@ -312,6 +434,8 @@ public final class PGuidePostsProtos {
maxLength_ = 0;
encodedGuidePostsCount_ = 0;
encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY;
+ byteCounts_ = java.util.Collections.emptyList();
+ rowCounts_ = java.util.Collections.emptyList();
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -343,6 +467,12 @@ public final class PGuidePostsProtos {
if (((bitField0_ & 0x00000010) == 0x00000010)) {
output.writeBytes(6, encodedGuidePosts_);
}
+ for (int i = 0; i < byteCounts_.size(); i++) {
+ output.writeInt64(7, byteCounts_.get(i));
+ }
+ for (int i = 0; i < rowCounts_.size(); i++) {
+ output.writeInt64(8, rowCounts_.get(i));
+ }
getUnknownFields().writeTo(output);
}
@@ -381,6 +511,24 @@ public final class PGuidePostsProtos {
size += com.google.protobuf.CodedOutputStream
.computeBytesSize(6, encodedGuidePosts_);
}
+ {
+ int dataSize = 0;
+ for (int i = 0; i < byteCounts_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeInt64SizeNoTag(byteCounts_.get(i));
+ }
+ size += dataSize;
+ size += 1 * getByteCountsList().size();
+ }
+ {
+ int dataSize = 0;
+ for (int i = 0; i < rowCounts_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeInt64SizeNoTag(rowCounts_.get(i));
+ }
+ size += dataSize;
+ size += 1 * getRowCountsList().size();
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -431,6 +579,10 @@ public final class PGuidePostsProtos {
result = result && getEncodedGuidePosts()
.equals(other.getEncodedGuidePosts());
}
+ result = result && getByteCountsList()
+ .equals(other.getByteCountsList());
+ result = result && getRowCountsList()
+ .equals(other.getRowCountsList());
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -468,6 +620,14 @@ public final class PGuidePostsProtos {
hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER;
hash = (53 * hash) + getEncodedGuidePosts().hashCode();
}
+ if (getByteCountsCount() > 0) {
+ hash = (37 * hash) + BYTECOUNTS_FIELD_NUMBER;
+ hash = (53 * hash) + getByteCountsList().hashCode();
+ }
+ if (getRowCountsCount() > 0) {
+ hash = (37 * hash) + ROWCOUNTS_FIELD_NUMBER;
+ hash = (53 * hash) + getRowCountsList().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -589,6 +749,10 @@ public final class PGuidePostsProtos {
bitField0_ = (bitField0_ & ~0x00000010);
encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY;
bitField0_ = (bitField0_ & ~0x00000020);
+ byteCounts_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ rowCounts_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000080);
return this;
}
@@ -642,6 +806,16 @@ public final class PGuidePostsProtos {
to_bitField0_ |= 0x00000010;
}
result.encodedGuidePosts_ = encodedGuidePosts_;
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ byteCounts_ = java.util.Collections.unmodifiableList(byteCounts_);
+ bitField0_ = (bitField0_ & ~0x00000040);
+ }
+ result.byteCounts_ = byteCounts_;
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ rowCounts_ = java.util.Collections.unmodifiableList(rowCounts_);
+ bitField0_ = (bitField0_ & ~0x00000080);
+ }
+ result.rowCounts_ = rowCounts_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -683,6 +857,26 @@ public final class PGuidePostsProtos {
if (other.hasEncodedGuidePosts()) {
setEncodedGuidePosts(other.getEncodedGuidePosts());
}
+ if (!other.byteCounts_.isEmpty()) {
+ if (byteCounts_.isEmpty()) {
+ byteCounts_ = other.byteCounts_;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ } else {
+ ensureByteCountsIsMutable();
+ byteCounts_.addAll(other.byteCounts_);
+ }
+ onChanged();
+ }
+ if (!other.rowCounts_.isEmpty()) {
+ if (rowCounts_.isEmpty()) {
+ rowCounts_ = other.rowCounts_;
+ bitField0_ = (bitField0_ & ~0x00000080);
+ } else {
+ ensureRowCountsIsMutable();
+ rowCounts_.addAll(other.rowCounts_);
+ }
+ onChanged();
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -950,6 +1144,138 @@ public final class PGuidePostsProtos {
return this;
}
+ // repeated int64 byteCounts = 7;
+ private java.util.List<java.lang.Long> byteCounts_ = java.util.Collections.emptyList();
+ private void ensureByteCountsIsMutable() {
+ if (!((bitField0_ & 0x00000040) == 0x00000040)) {
+ byteCounts_ = new java.util.ArrayList<java.lang.Long>(byteCounts_);
+ bitField0_ |= 0x00000040;
+ }
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public java.util.List<java.lang.Long>
+ getByteCountsList() {
+ return java.util.Collections.unmodifiableList(byteCounts_);
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public int getByteCountsCount() {
+ return byteCounts_.size();
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public long getByteCounts(int index) {
+ return byteCounts_.get(index);
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public Builder setByteCounts(
+ int index, long value) {
+ ensureByteCountsIsMutable();
+ byteCounts_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public Builder addByteCounts(long value) {
+ ensureByteCountsIsMutable();
+ byteCounts_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public Builder addAllByteCounts(
+ java.lang.Iterable<? extends java.lang.Long> values) {
+ ensureByteCountsIsMutable();
+ super.addAll(values, byteCounts_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int64 byteCounts = 7;</code>
+ */
+ public Builder clearByteCounts() {
+ byteCounts_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ onChanged();
+ return this;
+ }
+
+ // repeated int64 rowCounts = 8;
+ private java.util.List<java.lang.Long> rowCounts_ = java.util.Collections.emptyList();
+ private void ensureRowCountsIsMutable() {
+ if (!((bitField0_ & 0x00000080) == 0x00000080)) {
+ rowCounts_ = new java.util.ArrayList<java.lang.Long>(rowCounts_);
+ bitField0_ |= 0x00000080;
+ }
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public java.util.List<java.lang.Long>
+ getRowCountsList() {
+ return java.util.Collections.unmodifiableList(rowCounts_);
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public int getRowCountsCount() {
+ return rowCounts_.size();
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public long getRowCounts(int index) {
+ return rowCounts_.get(index);
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public Builder setRowCounts(
+ int index, long value) {
+ ensureRowCountsIsMutable();
+ rowCounts_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public Builder addRowCounts(long value) {
+ ensureRowCountsIsMutable();
+ rowCounts_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public Builder addAllRowCounts(
+ java.lang.Iterable<? extends java.lang.Long> values) {
+ ensureRowCountsIsMutable();
+ super.addAll(values, rowCounts_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int64 rowCounts = 8;</code>
+ */
+ public Builder clearRowCounts() {
+ rowCounts_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000080);
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:PGuidePosts)
}
@@ -975,13 +1301,13 @@ public final class PGuidePostsProtos {
descriptor;
static {
java.lang.String[] descriptorData = {
- "\n\021PGuidePosts.proto\"\224\001\n\013PGuidePosts\022\022\n\ng" +
+ "\n\021PGuidePosts.proto\"\273\001\n\013PGuidePosts\022\022\n\ng" +
"uidePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010ro" +
"wCount\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\036\n\026encod" +
"edGuidePostsCount\030\005 \001(\005\022\031\n\021encodedGuideP" +
- "osts\030\006 \001(\014BE\n(org.apache.phoenix.coproce" +
- "ssor.generatedB\021PGuidePostsProtosH\001\210\001\001\240\001" +
- "\001"
+ "osts\030\006 \001(\014\022\022\n\nbyteCounts\030\007 \003(\003\022\021\n\trowCou" +
+ "nts\030\010 \003(\003BE\n(org.apache.phoenix.coproces" +
+ "sor.generatedB\021PGuidePostsProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -993,7 +1319,7 @@ public final class PGuidePostsProtos {
internal_static_PGuidePosts_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_PGuidePosts_descriptor,
- new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "EncodedGuidePostsCount", "EncodedGuidePosts", });
+ new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "EncodedGuidePostsCount", "EncodedGuidePosts", "ByteCounts", "RowCounts", });
return null;
}
};
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index efb00a8..f4c570c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -121,7 +121,10 @@ public class ScanPlan extends BaseQueryPlan {
estRegionSize = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth, desc);
} else {
// Region size estimated based on total number of bytes divided by number of regions
- long totByteSize = gpsInfo.getByteCount();
+ long totByteSize = 0;
+ for (long byteCount : gpsInfo.getByteCounts()) {
+ totByteSize += byteCount;
+ }
estRegionSize = totByteSize / (gpsInfo.getGuidePostsCount()+1);
}
// TODO: configurable number of bytes?
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index abd31c0..bb90c16 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -1029,12 +1029,12 @@ public class PTableImpl implements PTable {
SortedMap<byte[], GuidePostsInfo> tableGuidePosts = new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR);
for (PTableProtos.PTableStats pTableStatsProto : table.getGuidePostsList()) {
PGuidePosts pGuidePosts = pTableStatsProto.getPGuidePosts();
- long guidePostsByteCount = pGuidePosts.getByteCount();
- long rowCount = pGuidePosts.getRowCount();
int maxLength = pGuidePosts.getMaxLength();
int guidePostsCount = pGuidePosts.getEncodedGuidePostsCount();
- GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount,
- new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getEncodedGuidePosts())), rowCount, maxLength, guidePostsCount);
+ GuidePostsInfo info = new GuidePostsInfo(pGuidePosts.getByteCountsList(),
+ new ImmutableBytesWritable(
+ HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getEncodedGuidePosts())),
+ pGuidePosts.getRowCountsList(), maxLength, guidePostsCount);
tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info);
}
PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp());
@@ -1142,17 +1142,16 @@ public class PTableImpl implements PTable {
for (Map.Entry<byte[], GuidePostsInfo> entry : table.getTableStats().getGuidePosts().entrySet()) {
PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder();
statsBuilder.setKey(ByteStringer.wrap(entry.getKey()));
- statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount());
statsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount());
PGuidePostsProtos.PGuidePosts.Builder guidePstsBuilder = PGuidePostsProtos.PGuidePosts.newBuilder();
- guidePstsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get()));
- guidePstsBuilder.setByteCount(entry.getValue().getByteCount());
- guidePstsBuilder.setRowCount(entry.getValue().getRowCount());
- guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength());
- guidePstsBuilder.setEncodedGuidePostsCount(entry.getValue().getGuidePostsCount());
- statsBuilder.setPGuidePosts(guidePstsBuilder);
- builder.addGuidePosts(statsBuilder.build());
- }
+ guidePstsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get()));
+ guidePstsBuilder.addAllByteCounts(entry.getValue().getByteCounts());
+ guidePstsBuilder.addAllRowCounts(entry.getValue().getRowCounts());
+ guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength());
+ guidePstsBuilder.setEncodedGuidePostsCount(entry.getValue().getGuidePostsCount());
+ statsBuilder.setPGuidePosts(guidePstsBuilder);
+ builder.addGuidePosts(statsBuilder.build());
+ }
builder.setStatsTimeStamp(table.getTableStats().getTimestamp());
if (table.getParentName() != null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
index da7d3a5..b8cc3f1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
@@ -17,6 +17,9 @@
*/
package org.apache.phoenix.schema.stats;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.util.ByteUtil;
/**
@@ -29,57 +32,69 @@ public class GuidePostsInfo {
* the total number of guidePosts for the table combining all the guidePosts per region per cf.
*/
private ImmutableBytesWritable guidePosts;
- /**
- * The bytecount that is flattened across the total number of guide posts.
- */
- private long byteCount = 0;
-
- /**
- * The rowCount that is flattened across the total number of guide posts.
- */
- private long rowCount = 0;
-
+
/**
* Maximum length of a guidePost collected
*/
private int maxLength;
- public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0,
- new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0);
+ public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(),
+ new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0);
public int getMaxLength() {
return maxLength;
}
-
+
+ /**
+ * Number of guidePosts
+ */
private int guidePostsCount;
/**
+ * The rowCounts of each guidePost traversed
+ */
+ private List<Long> rowCounts;
+
+ /**
+ * The bytecounts of each guidePost traversed
+ */
+ private List<Long> byteCounts;
+
+ public List<Long> getRowCounts() {
+ return rowCounts;
+ }
+
+ public List<Long> getByteCounts() {
+ return byteCounts;
+ }
+
+ /**
* Constructor that creates GuidePostsInfo per region
- * @param byteCount
+ *
+ * @param byteCounts
+ * The bytecounts of each guidePost traversed
* @param guidePosts
- * @param rowCount
+ * Prefix byte encoded guidePosts
+ * @param rowCounts
+ * The rowCounts of each guidePost traversed
+ * @param maxLength
+ * Maximum length of a guidePost collected
+ * @param guidePostsCount
+ * Number of guidePosts
*/
- public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long rowCount, int maxLength, int guidePostsCount) {
+ public GuidePostsInfo(List<Long> byteCounts, ImmutableBytesWritable guidePosts, List<Long> rowCounts, int maxLength,
+ int guidePostsCount) {
this.guidePosts = new ImmutableBytesWritable(guidePosts);
this.maxLength = maxLength;
- this.byteCount = byteCount;
- this.rowCount = rowCount;
this.guidePostsCount = guidePostsCount;
+ this.rowCounts = rowCounts;
+ this.byteCounts = byteCounts;
}
-
- public long getByteCount() {
- return byteCount;
- }
-
public ImmutableBytesWritable getGuidePosts() {
return guidePosts;
}
- public long getRowCount() {
- return this.rowCount;
- }
-
public int getGuidePostsCount() {
return guidePostsCount;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
index f3ada82..c85b1d6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java
@@ -19,9 +19,10 @@ package org.apache.phoenix.schema.stats;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PrefixByteCodec;
import org.apache.phoenix.util.PrefixByteEncoder;
@@ -33,25 +34,31 @@ import org.apache.phoenix.util.TrustedByteArrayOutputStream;
public class GuidePostsInfoBuilder {
private PrefixByteEncoder encoder;
- private byte[] lastRow;
+ private ImmutableBytesWritable lastRow;
private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY);
- private long byteCount = 0;
private int guidePostsCount;
/**
* The rowCount that is flattened across the total number of guide posts.
*/
private long rowCount = 0;
-
+
/**
* Maximum length of a guidePost collected
*/
private int maxLength;
private DataOutputStream output;
private TrustedByteArrayOutputStream stream;
-
- public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0,
- new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0);
+ private List<Long> rowCounts = new LinkedList<Long>();
+ private List<Long> byteCounts = new LinkedList<Long>();
+
+ public List<Long> getRowCounts() {
+ return rowCounts;
+ }
+
+ public List<Long> getByteCounts() {
+ return byteCounts;
+ }
public int getMaxLength() {
return maxLength;
@@ -60,7 +67,7 @@ public class GuidePostsInfoBuilder {
this.stream = new TrustedByteArrayOutputStream(1);
this.output = new DataOutputStream(stream);
this.encoder=new PrefixByteEncoder();
- lastRow = ByteUtil.EMPTY_BYTE_ARRAY;
+ lastRow = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY);
}
/**
@@ -71,14 +78,14 @@ public class GuidePostsInfoBuilder {
* @return
* @throws IOException
*/
- public boolean addGuidePosts( byte[] row, long byteCount, long rowCount) {
- if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) {
+ public boolean addGuidePosts(ImmutableBytesWritable row, long byteCount, long rowCount) {
+ if (row.getLength() != 0 && lastRow.compareTo(row) < 0) {
try {
- encoder.encode(output, row, 0, row.length);
- this.byteCount += byteCount;
+ encoder.encode(output, row.get(), row.getOffset(), row.getLength());
+ rowCounts.add(rowCount);
+ byteCounts.add(byteCount);
this.guidePostsCount++;
this.maxLength = encoder.getMaxLength();
- this.rowCount += rowCount;
lastRow = row;
return true;
} catch (IOException e) {
@@ -87,13 +94,17 @@ public class GuidePostsInfoBuilder {
}
return false;
}
-
- public boolean addGuidePosts(byte[] row){
- return addGuidePosts(row, 0, 0);
+
+ public boolean addGuidePosts(byte[] row) {
+ return addGuidePosts(new ImmutableBytesWritable(row), 0, 0);
+ }
+
+ public boolean addGuidePosts(byte[] row, long byteCount) {
+ return addGuidePosts(new ImmutableBytesWritable(row), byteCount, 0);
}
- public boolean addGuidePosts(byte[] row, long byteCount){
- return addGuidePosts(row, byteCount, 0);
+ public boolean addGuidePosts(byte[] row, long byteCount, long rowCount) {
+ return addGuidePosts(new ImmutableBytesWritable(row), byteCount, 0);
}
private void close() {
@@ -102,12 +113,22 @@ public class GuidePostsInfoBuilder {
public GuidePostsInfo build() {
this.guidePosts.set(stream.getBuffer(), 0, stream.size());
- GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, this.guidePostsCount);
+ GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCounts, this.guidePosts, this.rowCounts,
+ this.maxLength, this.guidePostsCount);
this.close();
return guidePostsInfo;
}
+
public void incrementRowCount() {
- this.rowCount++;
+ this.rowCount++;
+ }
+
+ public void resetRowCount() {
+ this.rowCount = 0;
+ }
+
+ public long getRowCount() {
+ return rowCount;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 3462f22..75e0aa0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -41,7 +41,6 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
-import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -160,10 +159,6 @@ public class StatisticsCollector {
*/
public void collectStatistics(final List<Cell> results) {
Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
- List<GuidePostsInfoBuilder> rowTracker = null;
- if (cachedGps == null) {
- rowTracker = new ArrayList<GuidePostsInfoBuilder>();
- }
for (Cell cell : results) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
@@ -179,29 +174,23 @@ public class StatisticsCollector {
}
if (famMap.get(cfKey) == null) {
famMap.put(cfKey, true);
- rowTracker.add(gps.getSecond());
+ gps.getSecond().incrementRowCount();
}
} else {
gps = cachedGps;
+ cachedGps.getSecond().incrementRowCount();
}
int kvLength = kv.getLength();
long byteCount = gps.getFirst() + kvLength;
gps.setFirst(byteCount);
if (byteCount >= guidepostDepth) {
- byte[] row = ByteUtil.copyKeyBytesIfNecessary(
- new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()));
- if (gps.getSecond().addGuidePosts(row, byteCount)) {
+ ImmutableBytesWritable row = new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
+ if (gps.getSecond().addGuidePosts(row, byteCount, gps.getSecond().getRowCount())) {
gps.setFirst(0l);
+ gps.getSecond().resetRowCount();
}
}
}
- if (cachedGps == null) {
- for (GuidePostsInfoBuilder s : rowTracker) {
- s.incrementRowCount();
- }
- } else {
- cachedGps.getSecond().incrementRowCount();
- }
}
public InternalScanner createCompactionScanner(RegionCoprocessorEnvironment env, Store store, InternalScanner s,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index d03af7a..dee5714 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -140,28 +140,28 @@ public class StatisticsWriter implements Closeable {
}
GuidePostsInfo gps = tracker.getGuidePosts(cfKey);
if (gps != null) {
- boolean rowColumnAdded = false;
+ List<Long> byteCounts = gps.getByteCounts();
+ List<Long> rowCounts = gps.getRowCounts();
ImmutableBytesWritable keys = gps.getGuidePosts();
ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength());
DataInput input = new DataInputStream(stream);
PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength());
+ int guidePostCount = 0;
try {
while (true) {
ImmutableBytesWritable ptr = decoder.decode(input);
byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr);
Put put = new Put(prefix);
- if (!rowColumnAdded) {
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
- timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount()));
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
- PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
- PLong.INSTANCE.toBytes(gps.getRowCount()));
- rowColumnAdded = true;
- }
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
+ timeStamp, PLong.INSTANCE.toBytes(byteCounts.get(guidePostCount)));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp,
+ PLong.INSTANCE.toBytes(rowCounts.get(guidePostCount)));
// Add our empty column value so queries behave correctly
put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp,
ByteUtil.EMPTY_BYTE_ARRAY);
mutations.add(put);
+ guidePostCount++;
}
} catch (EOFException e) { // Ignore as this signifies we're done
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d2fcd21d/phoenix-protocol/src/main/PGuidePosts.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PGuidePosts.proto b/phoenix-protocol/src/main/PGuidePosts.proto
index 14de2eb..99d2656 100644
--- a/phoenix-protocol/src/main/PGuidePosts.proto
+++ b/phoenix-protocol/src/main/PGuidePosts.proto
@@ -30,4 +30,6 @@ message PGuidePosts {
optional int32 maxLength = 4;
optional int32 encodedGuidePostsCount = 5;
optional bytes encodedGuidePosts = 6;
+ repeated int64 byteCounts = 7;
+ repeated int64 rowCounts = 8;
}
\ No newline at end of file