You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/10/11 00:44:02 UTC
git commit: PHOENIX-1333 Store statistics guideposts as VARBINARY
(ramkrishna.s.vasudevan)
Repository: phoenix
Updated Branches:
refs/heads/4.0 71d6d1a1e -> aae480ad9
PHOENIX-1333 Store statistics guideposts as VARBINARY (ramkrishna.s.vasudevan)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/aae480ad
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/aae480ad
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/aae480ad
Branch: refs/heads/4.0
Commit: aae480ad904390e51ffc31e6b7fb1d78f71b780d
Parents: 71d6d1a
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Oct 10 15:49:22 2014 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Oct 10 15:49:22 2014 -0700
----------------------------------------------------------------------
.../coprocessor/generated/PTableProtos.java | 310 +++++++++++++++++--
.../phoenix/iterate/ParallelIterators.java | 16 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 4 +
.../apache/phoenix/query/QueryConstants.java | 6 +-
.../org/apache/phoenix/schema/PTableImpl.java | 16 +-
.../phoenix/schema/stats/GuidePostsInfo.java | 144 +++++++++
.../phoenix/schema/stats/PTableStats.java | 5 +-
.../phoenix/schema/stats/PTableStatsImpl.java | 18 +-
.../schema/stats/StatisticsCollector.java | 56 ++--
.../phoenix/schema/stats/StatisticsUtil.java | 37 +--
.../phoenix/schema/stats/StatisticsWriter.java | 10 +-
phoenix-protocol/src/main/PTable.proto | 3 +
12 files changed, 521 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index 866870f..f1b3be1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@ -1601,6 +1601,36 @@ public final class PTableProtos {
* <code>repeated bytes values = 2;</code>
*/
com.google.protobuf.ByteString getValues(int index);
+
+ // optional int64 guidePostsByteCount = 3;
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ boolean hasGuidePostsByteCount();
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ long getGuidePostsByteCount();
+
+ // optional int64 keyBytesCount = 4;
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ boolean hasKeyBytesCount();
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ long getKeyBytesCount();
+
+ // optional int32 guidePostsCount = 5;
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ boolean hasGuidePostsCount();
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ int getGuidePostsCount();
}
/**
* Protobuf type {@code PTableStats}
@@ -1666,6 +1696,21 @@ public final class PTableProtos {
values_.add(input.readBytes());
break;
}
+ case 24: {
+ bitField0_ |= 0x00000002;
+ guidePostsByteCount_ = input.readInt64();
+ break;
+ }
+ case 32: {
+ bitField0_ |= 0x00000004;
+ keyBytesCount_ = input.readInt64();
+ break;
+ }
+ case 40: {
+ bitField0_ |= 0x00000008;
+ guidePostsCount_ = input.readInt32();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -1748,9 +1793,60 @@ public final class PTableProtos {
return values_.get(index);
}
+ // optional int64 guidePostsByteCount = 3;
+ public static final int GUIDEPOSTSBYTECOUNT_FIELD_NUMBER = 3;
+ private long guidePostsByteCount_;
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ public boolean hasGuidePostsByteCount() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ public long getGuidePostsByteCount() {
+ return guidePostsByteCount_;
+ }
+
+ // optional int64 keyBytesCount = 4;
+ public static final int KEYBYTESCOUNT_FIELD_NUMBER = 4;
+ private long keyBytesCount_;
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ public boolean hasKeyBytesCount() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ public long getKeyBytesCount() {
+ return keyBytesCount_;
+ }
+
+ // optional int32 guidePostsCount = 5;
+ public static final int GUIDEPOSTSCOUNT_FIELD_NUMBER = 5;
+ private int guidePostsCount_;
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ public boolean hasGuidePostsCount() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ public int getGuidePostsCount() {
+ return guidePostsCount_;
+ }
+
private void initFields() {
key_ = com.google.protobuf.ByteString.EMPTY;
values_ = java.util.Collections.emptyList();
+ guidePostsByteCount_ = 0L;
+ keyBytesCount_ = 0L;
+ guidePostsCount_ = 0;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -1774,6 +1870,15 @@ public final class PTableProtos {
for (int i = 0; i < values_.size(); i++) {
output.writeBytes(2, values_.get(i));
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeInt64(3, guidePostsByteCount_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeInt64(4, keyBytesCount_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeInt32(5, guidePostsCount_);
+ }
getUnknownFields().writeTo(output);
}
@@ -1796,6 +1901,18 @@ public final class PTableProtos {
size += dataSize;
size += 1 * getValuesList().size();
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(3, guidePostsByteCount_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(4, keyBytesCount_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(5, guidePostsCount_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -1826,6 +1943,21 @@ public final class PTableProtos {
}
result = result && getValuesList()
.equals(other.getValuesList());
+ result = result && (hasGuidePostsByteCount() == other.hasGuidePostsByteCount());
+ if (hasGuidePostsByteCount()) {
+ result = result && (getGuidePostsByteCount()
+ == other.getGuidePostsByteCount());
+ }
+ result = result && (hasKeyBytesCount() == other.hasKeyBytesCount());
+ if (hasKeyBytesCount()) {
+ result = result && (getKeyBytesCount()
+ == other.getKeyBytesCount());
+ }
+ result = result && (hasGuidePostsCount() == other.hasGuidePostsCount());
+ if (hasGuidePostsCount()) {
+ result = result && (getGuidePostsCount()
+ == other.getGuidePostsCount());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1847,6 +1979,18 @@ public final class PTableProtos {
hash = (37 * hash) + VALUES_FIELD_NUMBER;
hash = (53 * hash) + getValuesList().hashCode();
}
+ if (hasGuidePostsByteCount()) {
+ hash = (37 * hash) + GUIDEPOSTSBYTECOUNT_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getGuidePostsByteCount());
+ }
+ if (hasKeyBytesCount()) {
+ hash = (37 * hash) + KEYBYTESCOUNT_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getKeyBytesCount());
+ }
+ if (hasGuidePostsCount()) {
+ hash = (37 * hash) + GUIDEPOSTSCOUNT_FIELD_NUMBER;
+ hash = (53 * hash) + getGuidePostsCount();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1960,6 +2104,12 @@ public final class PTableProtos {
bitField0_ = (bitField0_ & ~0x00000001);
values_ = java.util.Collections.emptyList();
bitField0_ = (bitField0_ & ~0x00000002);
+ guidePostsByteCount_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ keyBytesCount_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000008);
+ guidePostsCount_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000010);
return this;
}
@@ -1997,6 +2147,18 @@ public final class PTableProtos {
bitField0_ = (bitField0_ & ~0x00000002);
}
result.values_ = values_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.guidePostsByteCount_ = guidePostsByteCount_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.keyBytesCount_ = keyBytesCount_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.guidePostsCount_ = guidePostsCount_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -2026,6 +2188,15 @@ public final class PTableProtos {
}
onChanged();
}
+ if (other.hasGuidePostsByteCount()) {
+ setGuidePostsByteCount(other.getGuidePostsByteCount());
+ }
+ if (other.hasKeyBytesCount()) {
+ setKeyBytesCount(other.getKeyBytesCount());
+ }
+ if (other.hasGuidePostsCount()) {
+ setGuidePostsCount(other.getGuidePostsCount());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -2165,6 +2336,105 @@ public final class PTableProtos {
return this;
}
+ // optional int64 guidePostsByteCount = 3;
+ private long guidePostsByteCount_ ;
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ public boolean hasGuidePostsByteCount() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ public long getGuidePostsByteCount() {
+ return guidePostsByteCount_;
+ }
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ public Builder setGuidePostsByteCount(long value) {
+ bitField0_ |= 0x00000004;
+ guidePostsByteCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int64 guidePostsByteCount = 3;</code>
+ */
+ public Builder clearGuidePostsByteCount() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ guidePostsByteCount_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional int64 keyBytesCount = 4;
+ private long keyBytesCount_ ;
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ public boolean hasKeyBytesCount() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ public long getKeyBytesCount() {
+ return keyBytesCount_;
+ }
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ public Builder setKeyBytesCount(long value) {
+ bitField0_ |= 0x00000008;
+ keyBytesCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int64 keyBytesCount = 4;</code>
+ */
+ public Builder clearKeyBytesCount() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ keyBytesCount_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional int32 guidePostsCount = 5;
+ private int guidePostsCount_ ;
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ public boolean hasGuidePostsCount() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ public int getGuidePostsCount() {
+ return guidePostsCount_;
+ }
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ public Builder setGuidePostsCount(int value) {
+ bitField0_ |= 0x00000010;
+ guidePostsCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int32 guidePostsCount = 5;</code>
+ */
+ public Builder clearGuidePostsCount() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ guidePostsCount_ = 0;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:PTableStats)
}
@@ -5649,24 +5919,26 @@ public final class PTableProtos {
"le\030\005 \001(\005\022\020\n\010nullable\030\006 \002(\010\022\020\n\010position\030\007" +
" \002(\005\022\021\n\tsortOrder\030\010 \002(\005\022\021\n\tarraySize\030\t \001" +
"(\005\022\024\n\014viewConstant\030\n \001(\014\022\026\n\016viewReferenc" +
- "ed\030\013 \001(\010\"*\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006" +
- "values\030\002 \003(\014\"\212\004\n\006PTable\022\027\n\017schemaNameByt" +
- "es\030\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttabl" +
- "eType\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004",
- " \001(\t\022\026\n\016sequenceNumber\030\005 \002(\003\022\021\n\ttimeStam" +
- "p\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 \001(\014\022\021\n\tbucketNu" +
- "m\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132\010.PColumn\022\030\n\007in" +
- "dexes\030\n \003(\0132\007.PTable\022\027\n\017isImmutableRows\030" +
- "\013 \002(\010\022 \n\nguidePosts\030\014 \003(\0132\014.PTableStats\022" +
- "\032\n\022dataTableNameBytes\030\r \001(\014\022\031\n\021defaultFa" +
- "milyName\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013mu" +
- "ltiTenant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rvie" +
- "wStatement\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022" +
- "\020\n\010tenantId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021",
- "\n\tindexType\030\026 \001(\014*A\n\nPTableType\022\n\n\006SYSTE" +
- "M\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JO" +
- "IN\020\004B@\n(org.apache.phoenix.coprocessor.g" +
- "eneratedB\014PTableProtosH\001\210\001\001\240\001\001"
+ "ed\030\013 \001(\010\"w\n\013PTableStats\022\013\n\003key\030\001 \002(\014\022\016\n\006" +
+ "values\030\002 \003(\014\022\033\n\023guidePostsByteCount\030\003 \001(" +
+ "\003\022\025\n\rkeyBytesCount\030\004 \001(\003\022\027\n\017guidePostsCo" +
+ "unt\030\005 \001(\005\"\212\004\n\006PTable\022\027\n\017schemaNameBytes\030",
+ "\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttableTy" +
+ "pe\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004 \001(" +
+ "\t\022\026\n\016sequenceNumber\030\005 \002(\003\022\021\n\ttimeStamp\030\006" +
+ " \002(\003\022\023\n\013pkNameBytes\030\007 \001(\014\022\021\n\tbucketNum\030\010" +
+ " \002(\005\022\031\n\007columns\030\t \003(\0132\010.PColumn\022\030\n\007index" +
+ "es\030\n \003(\0132\007.PTable\022\027\n\017isImmutableRows\030\013 \002" +
+ "(\010\022 \n\nguidePosts\030\014 \003(\0132\014.PTableStats\022\032\n\022" +
+ "dataTableNameBytes\030\r \001(\014\022\031\n\021defaultFamil" +
+ "yName\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013multi" +
+ "Tenant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rviewSt",
+ "atement\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020\n\010" +
+ "tenantId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n\ti" +
+ "ndexType\030\026 \001(\014*A\n\nPTableType\022\n\n\006SYSTEM\020\000" +
+ "\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020" +
+ "\004B@\n(org.apache.phoenix.coprocessor.gene" +
+ "ratedB\014PTableProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -5684,7 +5956,7 @@ public final class PTableProtos {
internal_static_PTableStats_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_PTableStats_descriptor,
- new java.lang.String[] { "Key", "Values", });
+ new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", });
internal_static_PTable_descriptor =
getDescriptor().getMessageTypes().get(2);
internal_static_PTable_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 8ec5215..a55ae70 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -61,6 +61,7 @@ import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.trace.util.Tracing;
import org.apache.phoenix.util.ByteUtil;
@@ -317,19 +318,26 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
List<byte[]> gps = null;
PTable table = getTable();
- Map<byte[],List<byte[]>> guidePostMap = tableStats.getGuidePosts();
+ Map<byte[],GuidePostsInfo> guidePostMap = tableStats.getGuidePosts();
byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable());
if (table.getColumnFamilies().isEmpty()) {
// For sure we can get the defaultCF from the table
- gps = guidePostMap.get(defaultCF);
+ if (guidePostMap.get(defaultCF) != null) {
+ gps = guidePostMap.get(defaultCF).getGuidePosts();
+ }
} else {
Scan scan = context.getScan();
if (scan.getFamilyMap().size() > 0 && !scan.getFamilyMap().containsKey(defaultCF)) {
// If default CF is not used in scan, use first CF referenced in scan
- gps = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next());
+ GuidePostsInfo guidePostsInfo = guidePostMap.get(scan.getFamilyMap().keySet().iterator().next());
+ if (guidePostsInfo != null) {
+ gps = guidePostsInfo.getGuidePosts();
+ }
} else {
// Otherwise, favor use of default CF.
- gps = guidePostMap.get(defaultCF);
+ if (guidePostMap.get(defaultCF) != null) {
+ gps = guidePostMap.get(defaultCF).getGuidePosts();
+ }
}
}
if (gps == null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 7afeb6e..9dd41f3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -233,6 +233,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho
public static final byte[] REGION_NAME_BYTES = Bytes.toBytes(REGION_NAME);
public static final String GUIDE_POSTS = "GUIDE_POSTS";
public static final byte[] GUIDE_POSTS_BYTES = Bytes.toBytes(GUIDE_POSTS);
+ public static final String GUIDE_POSTS_COUNT = "GUIDE_POSTS_COUNT";
+ public static final byte[] GUIDE_POSTS_COUNT_BYTES = Bytes.toBytes(GUIDE_POSTS_COUNT);
+ public static final String GUIDE_POSTS_WIDTH = "GUIDE_POSTS_WIDTH";
+ public static final byte[] GUIDE_POSTS_WIDTH_BYTES = Bytes.toBytes(GUIDE_POSTS_WIDTH);
public static final String MIN_KEY = "MIN_KEY";
public static final byte[] MIN_KEY_BYTES = Bytes.toBytes(MIN_KEY);
public static final String MAX_KEY = "MAX_KEY";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 84bb516..4ad3159 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -35,6 +35,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_COUNT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
@@ -235,7 +237,9 @@ public interface QueryConstants {
PHYSICAL_NAME + " VARCHAR NOT NULL," +
COLUMN_FAMILY + " VARCHAR," +
REGION_NAME + " VARCHAR," +
- GUIDE_POSTS + " VARBINARY[]," +
+ GUIDE_POSTS_COUNT + " BIGINT," +
+ GUIDE_POSTS + " VARBINARY," +
+ GUIDE_POSTS_WIDTH + " BIGINT," +
MIN_KEY + " VARBINARY," +
MAX_KEY + " VARBINARY," +
LAST_STATS_UPDATE_TIME+ " DATE, "+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index fdafc59..8f88e51 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -47,6 +47,7 @@ import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.protobuf.ProtobufUtil;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.schema.stats.PTableStatsImpl;
import org.apache.phoenix.util.ByteUtil;
@@ -379,7 +380,8 @@ public class PTableImpl implements PTable {
}
this.parentTableName = parentTableName;
- this.parentName = parentTableName == null ? null : PNameFactory.newName(SchemaUtil.getTableName(schemaName.getString(), parentTableName.getString()));
+ this.parentName = parentTableName == null ? null : PNameFactory.newName(SchemaUtil.getTableName(
+ schemaName.getString(), parentTableName.getString()));
estimatedSize += PNameFactory.getEstimatedSize(this.parentName);
this.physicalNames = physicalNames == null ? ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames);
@@ -712,7 +714,7 @@ public class PTableImpl implements PTable {
public long getTimeStamp() {
return timeStamp;
}
-
+
@Override
public PColumn getPKColumn(String name) throws ColumnNotFoundException {
List<PColumn> columns = columnsByName.get(name);
@@ -869,13 +871,14 @@ public class PTableImpl implements PTable {
}
boolean isImmutableRows = table.getIsImmutableRows();
- SortedMap<byte[], List<byte[]>> tableGuidePosts = new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR);
+ SortedMap<byte[], GuidePostsInfo> tableGuidePosts = new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR);
for (PTableProtos.PTableStats pTableStatsProto : table.getGuidePostsList()) {
List<byte[]> value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount());
for (int j = 0; j < pTableStatsProto.getValuesCount(); j++) {
value.add(pTableStatsProto.getValues(j).toByteArray());
}
- tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), value);
+ GuidePostsInfo info = new GuidePostsInfo(pTableStatsProto.getGuidePostsByteCount(), value);
+ tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info);
}
PTableStats stats = new PTableStatsImpl(tableGuidePosts);
@@ -962,12 +965,13 @@ public class PTableImpl implements PTable {
}
builder.setIsImmutableRows(table.isImmutableRows());
- for (Map.Entry<byte[], List<byte[]>> entry : table.getTableStats().getGuidePosts().entrySet()) {
+ for (Map.Entry<byte[], GuidePostsInfo> entry : table.getTableStats().getGuidePosts().entrySet()) {
PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder();
statsBuilder.setKey(HBaseZeroCopyByteString.wrap(entry.getKey()));
- for (byte[] stat : entry.getValue()) {
+ for (byte[] stat : entry.getValue().getGuidePosts()) {
statsBuilder.addValues(HBaseZeroCopyByteString.wrap(stat));
}
+ statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount());
builder.addGuidePosts(statsBuilder.build());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
new file mode 100644
index 0000000..6484349
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java
@@ -0,0 +1,144 @@
+package org.apache.phoenix.schema.stats;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.phoenix.util.TrustedByteArrayOutputStream;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * A simple POJO class that holds the information related to GuidePosts serDe.
+ */
+public class GuidePostsInfo {
+ private long byteCount; // Number of bytes traversed in the region
+ private long keyByteSize; // Total number of bytes in keys stored in guidePosts
+ private List<byte[]> guidePosts;
+
+ public GuidePostsInfo(long byteCount, List<byte[]> guidePosts) {
+ this.byteCount = byteCount;
+ this.guidePosts = ImmutableList.copyOf(guidePosts);
+ int size = 0;
+ for (byte[] key : guidePosts) {
+ size += key.length;
+ }
+ this.keyByteSize = size;
+ }
+
+ public boolean addGuidePost(byte[] row, long byteCount) {
+ if (guidePosts.isEmpty() || Bytes.compareTo(row, guidePosts.get(guidePosts.size() - 1)) > 0) {
+ List<byte[]> newGuidePosts = Lists.newArrayListWithExpectedSize(this.getGuidePosts().size() + 1);
+ newGuidePosts.addAll(guidePosts);
+ newGuidePosts.add(row);
+ this.guidePosts = ImmutableList.copyOf(newGuidePosts);
+ this.byteCount += byteCount;
+ this.keyByteSize += row.length;
+ return true;
+ }
+ return false;
+ }
+
+ public void combine(GuidePostsInfo oldInfo) {
+ // FIXME: I don't think we need to do a merge sort here, as the keys won't be interleaved.
+ // We just need to concatenate them in the correct way.
+ this.guidePosts = ImmutableList.copyOf(Iterators.mergeSorted(ImmutableList.of(this.getGuidePosts().iterator(), oldInfo.getGuidePosts().iterator()), Bytes.BYTES_COMPARATOR));
+ this.byteCount += oldInfo.getByteCount();
+ this.keyByteSize += oldInfo.keyByteSize;
+ }
+
+ public long getByteCount() {
+ return byteCount;
+ }
+
+ public List<byte[]> getGuidePosts() {
+ return guidePosts;
+ }
+
+ public static GuidePostsInfo fromBytes(byte[] buf, int offset, int l) {
+ try {
+ ByteArrayInputStream bytesIn = new ByteArrayInputStream(buf, offset, l);
+ try {
+ DataInputStream in = new DataInputStream(bytesIn);
+ try {
+ long byteCount = in.readLong();
+ int guidepostsCount = in.readInt();
+ List<byte[]> guidePosts = Lists.newArrayListWithExpectedSize(guidepostsCount);
+ if (guidepostsCount > 0) {
+ for (int i = 0; i < guidepostsCount; i++) {
+ int length = WritableUtils.readVInt(in);
+ byte[] gp = new byte[length];
+ in.read(gp);
+ if (gp.length != 0) {
+ guidePosts.add(gp);
+ }
+ }
+ }
+ return new GuidePostsInfo(byteCount, guidePosts);
+ } catch (IOException e) {
+ throw new RuntimeException(e); // not possible
+ } finally {
+ try {
+ in.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e); // not possible
+ }
+ }
+ } finally {
+ bytesIn.close();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e); // not possible
+ }
+ }
+
+ public byte[] toBytes() {
+ int size = guidePosts.size();
+ // Serialize the number of bytes traversed, number of key bytes in the region,
+ // number of guideposts for that family, <<guidepostSize><guidePostsArray>,<guidePostsSize> <guidePostArray>>
+ // We will lose precision here?
+ TrustedByteArrayOutputStream bs = new TrustedByteArrayOutputStream(
+ (int)(Bytes.SIZEOF_LONG + Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT + this.keyByteSize + (WritableUtils
+ .getVIntSize(size) * size)));
+ DataOutputStream os = new DataOutputStream(bs);
+ try {
+ os.writeLong(this.getByteCount());
+ os.writeInt(size);
+ for (byte[] element : guidePosts) {
+ WritableUtils.writeVInt(os, element.length);
+ os.write(element);
+ }
+ return bs.toByteArray();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe); // not possible
+ } finally {
+ try {
+ os.close();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe); // not possible
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
index 0782a2b..3745487 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java
@@ -17,7 +17,6 @@
*/
package org.apache.phoenix.schema.stats;
-import java.util.List;
import java.util.SortedMap;
import com.google.common.collect.ImmutableSortedMap;
@@ -30,7 +29,7 @@ import com.google.common.collect.ImmutableSortedMap;
public interface PTableStats {
public static final PTableStats EMPTY_STATS = new PTableStats() {
@Override
- public SortedMap<byte[], List<byte[]>> getGuidePosts() {
+ public SortedMap<byte[], GuidePostsInfo> getGuidePosts() {
return ImmutableSortedMap.of();
}
@@ -45,7 +44,7 @@ public interface PTableStats {
* Returns a tree map of the guide posts collected against a column family
* @return
*/
- SortedMap<byte[], List<byte[]>> getGuidePosts();
+ SortedMap<byte[], GuidePostsInfo> getGuidePosts();
int getEstimatedSize();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
index 5d7d2ac..dcf7b00 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java
@@ -31,30 +31,31 @@ import com.sun.istack.NotNull;
* Implementation for PTableStats.
*/
public class PTableStatsImpl implements PTableStats {
- private final SortedMap<byte[], List<byte[]>> guidePosts;
+ private final SortedMap<byte[], GuidePostsInfo> guidePosts;
private final int estimatedSize;
public PTableStatsImpl() {
- this(new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR));
+ this(new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR));
}
- public PTableStatsImpl(@NotNull SortedMap<byte[], List<byte[]>> guidePosts) {
+ public PTableStatsImpl(@NotNull SortedMap<byte[], GuidePostsInfo> guidePosts) {
this.guidePosts = guidePosts;
int estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.INT_SIZE + SizedUtil.sizeOfTreeMap(guidePosts.size());
- for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) {
+ for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) {
byte[] cf = entry.getKey();
estimatedSize += SizedUtil.ARRAY_SIZE + cf.length;
- List<byte[]> keys = entry.getValue();
+ List<byte[]> keys = entry.getValue().getGuidePosts();
estimatedSize += SizedUtil.sizeOfArrayList(keys.size());
for (byte[] key : keys) {
estimatedSize += SizedUtil.ARRAY_SIZE + key.length;
}
+ estimatedSize += SizedUtil.LONG_SIZE;
}
this.estimatedSize = estimatedSize;
}
@Override
- public SortedMap<byte[], List<byte[]>> getGuidePosts() {
+ public SortedMap<byte[], GuidePostsInfo> getGuidePosts() {
return guidePosts;
}
@@ -62,10 +63,10 @@ public class PTableStatsImpl implements PTableStats {
public String toString() {
StringBuilder buf = new StringBuilder();
buf.append("PTableStats [");
- for (Map.Entry<byte[], List<byte[]>> entry : guidePosts.entrySet()) {
+ for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) {
buf.append(Bytes.toStringBinary(entry.getKey()));
buf.append(":(");
- List<byte[]> keys = entry.getValue();
+ List<byte[]> keys = entry.getValue().getGuidePosts();
if (!keys.isEmpty()) {
for (byte[] key : keys) {
buf.append(Bytes.toStringBinary(key));
@@ -83,5 +84,4 @@ public class PTableStatsImpl implements PTableStats {
public int getEstimatedSize() {
return estimatedSize;
}
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 49e6b39..53bd18a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.schema.stats;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -26,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.TableName;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
@@ -46,8 +49,7 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.schema.PDataType;
-import org.apache.phoenix.schema.PhoenixArray;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.TimeKeeper;
import com.google.common.collect.Lists;
@@ -66,7 +68,8 @@ public class StatisticsCollector {
private Map<String, byte[]> minMap = Maps.newHashMap();
private Map<String, byte[]> maxMap = Maps.newHashMap();
private long guidepostDepth;
- private Map<String, Pair<Integer,List<byte[]>>> guidePostsMap = Maps.newHashMap();
+ private Map<String, Pair<Long,GuidePostsInfo>> guidePostsMap = Maps.newHashMap();
+ // Tracks the bytecount per family if it has reached the guidePostsDepth
private Map<ImmutableBytesPtr, Boolean> familyMap = Maps.newHashMap();
protected StatisticsWriter statsTable;
// Ensures that either analyze or compaction happens at any point of time.
@@ -75,11 +78,14 @@ public class StatisticsCollector {
public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException {
Configuration config = env.getConfiguration();
HTableInterface statsHTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES));
+ long maxFileSize = statsHTable.getTableDescriptor().getMaxFileSize();
+ if (maxFileSize <= 0) { // HBase brain dead API doesn't give you the "real" max file size if it's not set...
+ maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE;
+ }
guidepostDepth =
config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- statsHTable.getTableDescriptor().getMaxFileSize() /
- config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_GUIDE_POSTS_PER_REGION));
+ maxFileSize / config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_GUIDE_POSTS_PER_REGION));
// Get the stats table associated with the current table on which the CP is
// triggered
this.statsTable = StatisticsWriter.newWriter(statsHTable, tableName, clientTimeStamp);
@@ -280,8 +286,8 @@ public class StatisticsCollector {
familyMap.put(new ImmutableBytesPtr(cf), true);
String fam = Bytes.toString(cf);
- byte[] row = new ImmutableBytesPtr(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())
- .copyBytesIfNecessary();
+ byte[] row = ByteUtil.copyKeyBytesIfNecessary(
+ new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()));
if (!minMap.containsKey(fam) && !maxMap.containsKey(fam)) {
minMap.put(fam, row);
// Ideally the max key also should be added in this case
@@ -297,19 +303,17 @@ public class StatisticsCollector {
}
}
// TODO : This can be moved to an interface so that we could collect guide posts in different ways
- Pair<Integer,List<byte[]>> gps = guidePostsMap.get(fam);
+ Pair<Long,GuidePostsInfo> gps = guidePostsMap.get(fam);
if (gps == null) {
- gps = new Pair<Integer,List<byte[]>>(0, Lists.<byte[]>newArrayList());
+ gps = new Pair<Long,GuidePostsInfo>(0L,new GuidePostsInfo(0, Collections.<byte[]>emptyList()));
guidePostsMap.put(fam, gps);
}
- int byteCount = gps.getFirst() + kv.getLength();
+ int kvLength = kv.getLength();
+ long byteCount = gps.getFirst() + kvLength;
gps.setFirst(byteCount);
if (byteCount >= guidepostDepth) {
- // Prevent dups
- List<byte[]> gpsKeys = gps.getSecond();
- if (gpsKeys.isEmpty() || Bytes.compareTo(row, gpsKeys.get(gpsKeys.size()-1)) > 0) {
- gpsKeys.add(row);
- gps.setFirst(0); // Only reset count when adding guidepost
+ if (gps.getSecond().addGuidePost(row, byteCount)) {
+ gps.setFirst(0L);
}
}
}
@@ -324,22 +328,10 @@ public class StatisticsCollector {
return null;
}
- public byte[] getGuidePosts(String fam) {
- if (!guidePostsMap.isEmpty()) {
- Pair<Integer,List<byte[]>> gps = guidePostsMap.get(fam);
- if (gps != null) {
- List<byte[]> guidePosts = gps.getSecond();
- if (!guidePosts.isEmpty()) {
- byte[][] array = new byte[guidePosts.size()][];
- int i = 0;
- for (byte[] element : guidePosts) {
- array[i] = element;
- i++;
- }
- PhoenixArray phoenixArray = new PhoenixArray(PDataType.VARBINARY, array);
- return PDataType.VARBINARY_ARRAY.toBytes(phoenixArray);
- }
- }
+ public GuidePostsInfo getGuidePosts(String fam) {
+ Pair<Long,GuidePostsInfo> pair = guidePostsMap.get(fam);
+ if (pair != null) {
+ return pair.getSecond();
}
return null;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 5ef757c..b8d64bd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -20,8 +20,6 @@ import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
import java.util.TreeMap;
import org.apache.hadoop.hbase.Cell;
@@ -36,12 +34,8 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.schema.PDataType;
-import org.apache.phoenix.schema.PhoenixArray;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.MetaDataUtil;
-
-import com.google.common.collect.Lists;
/**
* Simple utility class for managing multiple key parts of the statistic
*/
@@ -71,45 +65,32 @@ public class StatisticsUtil {
return Arrays.copyOfRange(kv.getRowArray(), kv.getRowOffset(), kv.getRowOffset() + kv.getRowLength());
}
- public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, long clientTimeStamp) throws IOException {
+ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, long clientTimeStamp)
+ throws IOException {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
Scan s = MetaDataUtil.newTableRowsScan(tableNameBytes, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp);
s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_BYTES);
ResultScanner scanner = statsHTable.getScanner(s);
Result result = null;
- TreeMap<byte[], List<byte[]>> guidePostsPerCf = new TreeMap<byte[], List<byte[]>>(Bytes.BYTES_COMPARATOR);
+ TreeMap<byte[], GuidePostsInfo> guidePostsPerCf = new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR);
while ((result = scanner.next()) != null) {
CellScanner cellScanner = result.cellScanner();
while (cellScanner.advance()) {
Cell current = cellScanner.current();
int tableNameLength = tableNameBytes.length + 1;
int cfOffset = current.getRowOffset() + tableNameLength;
- int cfLength = getVarCharLength(current.getRowArray(), cfOffset, current.getRowLength() - tableNameLength);
+ int cfLength = getVarCharLength(current.getRowArray(), cfOffset, current.getRowLength()
+ - tableNameLength);
ptr.set(current.getRowArray(), cfOffset, cfLength);
byte[] cfName = ByteUtil.copyKeyBytesIfNecessary(ptr);
- PhoenixArray array = (PhoenixArray)PDataType.VARBINARY_ARRAY.toObject(current.getValueArray(), current.getValueOffset(), current
- .getValueLength());
- if (array != null && array.getDimensions() != 0) {
- List<byte[]> guidePosts = Lists.newArrayListWithExpectedSize(array.getDimensions());
- for (int j = 0; j < array.getDimensions(); j++) {
- byte[] gp = array.toBytes(j);
- if (gp.length != 0) {
- guidePosts.add(gp);
- }
- }
- List<byte[]> gps = guidePostsPerCf.put(cfName, guidePosts);
- if (gps != null) { // Add guidepost already there from other regions
- guidePosts.addAll(gps);
- }
+ GuidePostsInfo newInfo = GuidePostsInfo.fromBytes(current.getValueArray(), current.getValueOffset(), current.getValueLength());
+ GuidePostsInfo oldInfo = guidePostsPerCf.put(cfName, newInfo);
+ if (oldInfo != null) {
+ newInfo.combine(oldInfo);
}
}
}
if (!guidePostsPerCf.isEmpty()) {
- // Sort guideposts, as the order above will depend on the order we traverse
- // each region's worth of guideposts above.
- for (List<byte[]> gps : guidePostsPerCf.values()) {
- Collections.sort(gps, Bytes.BYTES_COMPARATOR);
- }
return new PTableStatsImpl(guidePostsPerCf);
}
return PTableStats.EMPTY_STATS;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
index ddfc2d6..6da135e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java
@@ -48,6 +48,7 @@ public class StatisticsWriter implements Closeable {
/**
* @param tableName TODO
* @param clientTimeStamp TODO
+ * @param guidepostDepth
* @param Configuration
* Configruation to update the stats table.
* @param primaryTableName
@@ -104,9 +105,14 @@ public class StatisticsWriter implements Closeable {
byte[] prefix = StatisticsUtil.getRowKey(tableName, PDataType.VARCHAR.toBytes(fam),
PDataType.VARCHAR.toBytes(regionName));
Put put = new Put(prefix);
- if (tracker.getGuidePosts(fam) != null) {
+ GuidePostsInfo gp = tracker.getGuidePosts(fam);
+ if (gp != null) {
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_COUNT_BYTES,
+ clientTimeStamp, PDataType.LONG.toBytes((gp.getGuidePosts().size())));
put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_BYTES,
- clientTimeStamp, (tracker.getGuidePosts(fam)));
+ clientTimeStamp, PDataType.VARBINARY.toBytes(gp.toBytes()));
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES,
+ clientTimeStamp, PDataType.LONG.toBytes(gp.getByteCount()));
}
put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.MIN_KEY_BYTES,
clientTimeStamp, PDataType.VARBINARY.toBytes(tracker.getMinKey(fam)));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aae480ad/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto
index 0edc046..b622a26 100644
--- a/phoenix-protocol/src/main/PTable.proto
+++ b/phoenix-protocol/src/main/PTable.proto
@@ -49,6 +49,9 @@ message PColumn {
message PTableStats {
required bytes key = 1;
repeated bytes values = 2;
+ optional int64 guidePostsByteCount = 3;
+ optional int64 keyBytesCount = 4;
+ optional int32 guidePostsCount = 5;
}
message PTable {