You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/25 14:12:16 UTC
[3/7] incubator-kylin git commit: KYLIN-942 support parallel scan for
grid table
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 2e6741b..225703d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -30,6 +30,31 @@ public final class CubeVisitProtos {
* <code>required bytes hbaseRawScan = 2;</code>
*/
com.google.protobuf.ByteString getHbaseRawScan();
+
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>
+ getHbaseColumnsToGTList();
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index);
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ int getHbaseColumnsToGTCount();
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
+ getHbaseColumnsToGTOrBuilderList();
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
+ int index);
}
/**
* Protobuf type {@code CubeVisitRequest}
@@ -87,48 +112,563 @@ public final class CubeVisitProtos {
gtScanRequest_ = input.readBytes();
break;
}
- case 18: {
- bitField0_ |= 0x00000002;
- hbaseRawScan_ = input.readBytes();
- break;
+ case 18: {
+ bitField0_ |= 0x00000002;
+ hbaseRawScan_ = input.readBytes();
+ break;
+ }
+ case 26: {
+ if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>();
+ mutable_bitField0_ |= 0x00000004;
+ }
+ hbaseColumnsToGT_.add(input.readMessage(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.PARSER, extensionRegistry));
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.class, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<CubeVisitRequest> PARSER =
+ new com.google.protobuf.AbstractParser<CubeVisitRequest>() {
+ public CubeVisitRequest parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new CubeVisitRequest(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<CubeVisitRequest> getParserForType() {
+ return PARSER;
+ }
+
+ public interface IntListOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // repeated int32 ints = 1;
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ java.util.List<java.lang.Integer> getIntsList();
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ int getIntsCount();
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ int getInts(int index);
+ }
+ /**
+ * Protobuf type {@code CubeVisitRequest.IntList}
+ */
+ public static final class IntList extends
+ com.google.protobuf.GeneratedMessage
+ implements IntListOrBuilder {
+ // Use IntList.newBuilder() to construct.
+ private IntList(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private IntList(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final IntList defaultInstance;
+ public static IntList getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public IntList getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private IntList(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ ints_ = new java.util.ArrayList<java.lang.Integer>();
+ mutable_bitField0_ |= 0x00000001;
+ }
+ ints_.add(input.readInt32());
+ break;
+ }
+ case 10: {
+ int length = input.readRawVarint32();
+ int limit = input.pushLimit(length);
+ if (!((mutable_bitField0_ & 0x00000001) == 0x00000001) && input.getBytesUntilLimit() > 0) {
+ ints_ = new java.util.ArrayList<java.lang.Integer>();
+ mutable_bitField0_ |= 0x00000001;
+ }
+ while (input.getBytesUntilLimit() > 0) {
+ ints_.add(input.readInt32());
+ }
+ input.popLimit(limit);
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ ints_ = java.util.Collections.unmodifiableList(ints_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_IntList_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_IntList_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.class, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<IntList> PARSER =
+ new com.google.protobuf.AbstractParser<IntList>() {
+ public IntList parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new IntList(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<IntList> getParserForType() {
+ return PARSER;
+ }
+
+ // repeated int32 ints = 1;
+ public static final int INTS_FIELD_NUMBER = 1;
+ private java.util.List<java.lang.Integer> ints_;
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public java.util.List<java.lang.Integer>
+ getIntsList() {
+ return ints_;
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public int getIntsCount() {
+ return ints_.size();
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public int getInts(int index) {
+ return ints_.get(index);
+ }
+
+ private void initFields() {
+ ints_ = java.util.Collections.emptyList();
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ for (int i = 0; i < ints_.size(); i++) {
+ output.writeInt32(1, ints_.get(i));
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ {
+ int dataSize = 0;
+ for (int i = 0; i < ints_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeInt32SizeNoTag(ints_.get(i));
+ }
+ size += dataSize;
+ size += 1 * getIntsList().size();
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList)) {
+ return super.equals(obj);
+ }
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList other = (org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList) obj;
+
+ boolean result = true;
+ result = result && getIntsList()
+ .equals(other.getIntsList());
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (getIntsCount() > 0) {
+ hash = (37 * hash) + INTS_FIELD_NUMBER;
+ hash = (53 * hash) + getIntsList().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code CubeVisitRequest.IntList}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_IntList_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_IntList_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.class, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder.class);
+ }
+
+ // Construct using org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ ints_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000001);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_IntList_descriptor;
+ }
+
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getDefaultInstanceForType() {
+ return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance();
+ }
+
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList build() {
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList buildPartial() {
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList result = new org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList(this);
+ int from_bitField0_ = bitField0_;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ ints_ = java.util.Collections.unmodifiableList(ints_);
+ bitField0_ = (bitField0_ & ~0x00000001);
+ }
+ result.ints_ = ints_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList) {
+ return mergeFrom((org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList other) {
+ if (other == org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance()) return this;
+ if (!other.ints_.isEmpty()) {
+ if (ints_.isEmpty()) {
+ ints_ = other.ints_;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ } else {
+ ensureIntsIsMutable();
+ ints_.addAll(other.ints_);
+ }
+ onChanged();
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
}
}
+ return this;
}
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_descriptor;
- }
+ private int bitField0_;
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.internal_static_CubeVisitRequest_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.class, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.Builder.class);
- }
+ // repeated int32 ints = 1;
+ private java.util.List<java.lang.Integer> ints_ = java.util.Collections.emptyList();
+ private void ensureIntsIsMutable() {
+ if (!((bitField0_ & 0x00000001) == 0x00000001)) {
+ ints_ = new java.util.ArrayList<java.lang.Integer>(ints_);
+ bitField0_ |= 0x00000001;
+ }
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public java.util.List<java.lang.Integer>
+ getIntsList() {
+ return java.util.Collections.unmodifiableList(ints_);
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public int getIntsCount() {
+ return ints_.size();
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public int getInts(int index) {
+ return ints_.get(index);
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public Builder setInts(
+ int index, int value) {
+ ensureIntsIsMutable();
+ ints_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public Builder addInts(int value) {
+ ensureIntsIsMutable();
+ ints_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public Builder addAllInts(
+ java.lang.Iterable<? extends java.lang.Integer> values) {
+ ensureIntsIsMutable();
+ super.addAll(values, ints_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated int32 ints = 1;</code>
+ */
+ public Builder clearInts() {
+ ints_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000001);
+ onChanged();
+ return this;
+ }
- public static com.google.protobuf.Parser<CubeVisitRequest> PARSER =
- new com.google.protobuf.AbstractParser<CubeVisitRequest>() {
- public CubeVisitRequest parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new CubeVisitRequest(input, extensionRegistry);
+ // @@protoc_insertion_point(builder_scope:CubeVisitRequest.IntList)
}
- };
- @java.lang.Override
- public com.google.protobuf.Parser<CubeVisitRequest> getParserForType() {
- return PARSER;
+ static {
+ defaultInstance = new IntList(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:CubeVisitRequest.IntList)
}
private int bitField0_;
@@ -164,9 +704,46 @@ public final class CubeVisitProtos {
return hbaseRawScan_;
}
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 3;
+ private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_;
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
+ return hbaseColumnsToGT_;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
+ getHbaseColumnsToGTOrBuilderList() {
+ return hbaseColumnsToGT_;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public int getHbaseColumnsToGTCount() {
+ return hbaseColumnsToGT_.size();
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
+ return hbaseColumnsToGT_.get(index);
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
+ int index) {
+ return hbaseColumnsToGT_.get(index);
+ }
+
private void initFields() {
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
+ hbaseColumnsToGT_ = java.util.Collections.emptyList();
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -194,6 +771,9 @@ public final class CubeVisitProtos {
if (((bitField0_ & 0x00000002) == 0x00000002)) {
output.writeBytes(2, hbaseRawScan_);
}
+ for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
+ output.writeMessage(3, hbaseColumnsToGT_.get(i));
+ }
getUnknownFields().writeTo(output);
}
@@ -211,6 +791,10 @@ public final class CubeVisitProtos {
size += com.google.protobuf.CodedOutputStream
.computeBytesSize(2, hbaseRawScan_);
}
+ for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(3, hbaseColumnsToGT_.get(i));
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -244,6 +828,8 @@ public final class CubeVisitProtos {
result = result && getHbaseRawScan()
.equals(other.getHbaseRawScan());
}
+ result = result && getHbaseColumnsToGTList()
+ .equals(other.getHbaseColumnsToGTList());
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -265,6 +851,10 @@ public final class CubeVisitProtos {
hash = (37 * hash) + HBASERAWSCAN_FIELD_NUMBER;
hash = (53 * hash) + getHbaseRawScan().hashCode();
}
+ if (getHbaseColumnsToGTCount() > 0) {
+ hash = (37 * hash) + HBASECOLUMNSTOGT_FIELD_NUMBER;
+ hash = (53 * hash) + getHbaseColumnsToGTList().hashCode();
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -366,6 +956,7 @@ public final class CubeVisitProtos {
}
private void maybeForceBuilderInitialization() {
if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getHbaseColumnsToGTFieldBuilder();
}
}
private static Builder create() {
@@ -378,6 +969,12 @@ public final class CubeVisitProtos {
bitField0_ = (bitField0_ & ~0x00000001);
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
bitField0_ = (bitField0_ & ~0x00000002);
+ if (hbaseColumnsToGTBuilder_ == null) {
+ hbaseColumnsToGT_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000004);
+ } else {
+ hbaseColumnsToGTBuilder_.clear();
+ }
return this;
}
@@ -414,6 +1011,15 @@ public final class CubeVisitProtos {
to_bitField0_ |= 0x00000002;
}
result.hbaseRawScan_ = hbaseRawScan_;
+ if (hbaseColumnsToGTBuilder_ == null) {
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
+ bitField0_ = (bitField0_ & ~0x00000004);
+ }
+ result.hbaseColumnsToGT_ = hbaseColumnsToGT_;
+ } else {
+ result.hbaseColumnsToGT_ = hbaseColumnsToGTBuilder_.build();
+ }
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -436,6 +1042,32 @@ public final class CubeVisitProtos {
if (other.hasHbaseRawScan()) {
setHbaseRawScan(other.getHbaseRawScan());
}
+ if (hbaseColumnsToGTBuilder_ == null) {
+ if (!other.hbaseColumnsToGT_.isEmpty()) {
+ if (hbaseColumnsToGT_.isEmpty()) {
+ hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ } else {
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.addAll(other.hbaseColumnsToGT_);
+ }
+ onChanged();
+ }
+ } else {
+ if (!other.hbaseColumnsToGT_.isEmpty()) {
+ if (hbaseColumnsToGTBuilder_.isEmpty()) {
+ hbaseColumnsToGTBuilder_.dispose();
+ hbaseColumnsToGTBuilder_ = null;
+ hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ hbaseColumnsToGTBuilder_ =
+ com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+ getHbaseColumnsToGTFieldBuilder() : null;
+ } else {
+ hbaseColumnsToGTBuilder_.addAllMessages(other.hbaseColumnsToGT_);
+ }
+ }
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -543,6 +1175,246 @@ public final class CubeVisitProtos {
return this;
}
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_ =
+ java.util.Collections.emptyList();
+ private void ensureHbaseColumnsToGTIsMutable() {
+ if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+ hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>(hbaseColumnsToGT_);
+ bitField0_ |= 0x00000004;
+ }
+ }
+
+ private com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder> hbaseColumnsToGTBuilder_;
+
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ return java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
+ } else {
+ return hbaseColumnsToGTBuilder_.getMessageList();
+ }
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public int getHbaseColumnsToGTCount() {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ return hbaseColumnsToGT_.size();
+ } else {
+ return hbaseColumnsToGTBuilder_.getCount();
+ }
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ return hbaseColumnsToGT_.get(index);
+ } else {
+ return hbaseColumnsToGTBuilder_.getMessage(index);
+ }
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder setHbaseColumnsToGT(
+ int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.set(index, value);
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.setMessage(index, value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder setHbaseColumnsToGT(
+ int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.set(index, builderForValue.build());
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.setMessage(index, builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder addHbaseColumnsToGT(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.add(value);
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.addMessage(value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder addHbaseColumnsToGT(
+ int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.add(index, value);
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.addMessage(index, value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder addHbaseColumnsToGT(
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.add(builderForValue.build());
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.addMessage(builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder addHbaseColumnsToGT(
+ int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.add(index, builderForValue.build());
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.addMessage(index, builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder addAllHbaseColumnsToGT(
+ java.lang.Iterable<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> values) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ ensureHbaseColumnsToGTIsMutable();
+ super.addAll(values, hbaseColumnsToGT_);
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.addAllMessages(values);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder clearHbaseColumnsToGT() {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ hbaseColumnsToGT_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000004);
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.clear();
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public Builder removeHbaseColumnsToGT(int index) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ ensureHbaseColumnsToGTIsMutable();
+ hbaseColumnsToGT_.remove(index);
+ onChanged();
+ } else {
+ hbaseColumnsToGTBuilder_.remove(index);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder getHbaseColumnsToGTBuilder(
+ int index) {
+ return getHbaseColumnsToGTFieldBuilder().getBuilder(index);
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
+ int index) {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ return hbaseColumnsToGT_.get(index); } else {
+ return hbaseColumnsToGTBuilder_.getMessageOrBuilder(index);
+ }
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
+ getHbaseColumnsToGTOrBuilderList() {
+ if (hbaseColumnsToGTBuilder_ != null) {
+ return hbaseColumnsToGTBuilder_.getMessageOrBuilderList();
+ } else {
+ return java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
+ }
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder() {
+ return getHbaseColumnsToGTFieldBuilder().addBuilder(
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder(
+ int index) {
+ return getHbaseColumnsToGTFieldBuilder().addBuilder(
+ index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
+ }
+ /**
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ */
+ public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder>
+ getHbaseColumnsToGTBuilderList() {
+ return getHbaseColumnsToGTFieldBuilder().getBuilderList();
+ }
+ private com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
+ getHbaseColumnsToGTFieldBuilder() {
+ if (hbaseColumnsToGTBuilder_ == null) {
+ hbaseColumnsToGTBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>(
+ hbaseColumnsToGT_,
+ ((bitField0_ & 0x00000004) == 0x00000004),
+ getParentForChildren(),
+ isClean());
+ hbaseColumnsToGT_ = null;
+ }
+ return hbaseColumnsToGTBuilder_;
+ }
+
// @@protoc_insertion_point(builder_scope:CubeVisitRequest)
}
@@ -2158,6 +3030,11 @@ public final class CubeVisitProtos {
com.google.protobuf.GeneratedMessage.FieldAccessorTable
internal_static_CubeVisitRequest_fieldAccessorTable;
private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_CubeVisitRequest_IntList_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_CubeVisitRequest_IntList_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
internal_static_CubeVisitResponse_descriptor;
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
@@ -2178,18 +3055,20 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"?\n\020Cu" +
- "beVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024\n" +
- "\014hbaseRawScan\030\002 \002(\014\"\304\001\n\021CubeVisitRespons" +
- "e\022\026\n\016compressedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\013" +
- "2\030.CubeVisitResponse.Stats\032n\n\005Stats\022\030\n\020s" +
- "erviceStartTime\030\001 \001(\003\022\026\n\016serviceEndTime\030" +
- "\002 \001(\003\022\027\n\017scannedRowCount\030\003 \001(\005\022\032\n\022aggreg" +
- "atedRowCount\030\004 \001(\0052F\n\020CubeVisitService\0222",
- "\n\tvisitCube\022\021.CubeVisitRequest\032\022.CubeVis" +
- "itResponseB`\nEorg.apache.kylin.storage.h" +
- "base.cube.v2.coprocessor.endpoint.genera" +
- "tedB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "/endpoint/protobuf/CubeVisit.proto\"\215\001\n\020C" +
+ "ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
+ "\n\014hbaseRawScan\030\002 \002(\014\0223\n\020hbaseColumnsToGT" +
+ "\030\003 \003(\0132\031.CubeVisitRequest.IntList\032\027\n\007Int" +
+ "List\022\014\n\004ints\030\001 \003(\005\"\304\001\n\021CubeVisitResponse" +
+ "\022\026\n\016compressedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132" +
+ "\030.CubeVisitResponse.Stats\032n\n\005Stats\022\030\n\020se" +
+ "rviceStartTime\030\001 \001(\003\022\026\n\016serviceEndTime\030\002",
+ " \001(\003\022\027\n\017scannedRowCount\030\003 \001(\005\022\032\n\022aggrega" +
+ "tedRowCount\030\004 \001(\0052F\n\020CubeVisitService\0222\n" +
+ "\tvisitCube\022\021.CubeVisitRequest\032\022.CubeVisi" +
+ "tResponseB`\nEorg.apache.kylin.storage.hb" +
+ "ase.cube.v2.coprocessor.endpoint.generat" +
+ "edB\017CubeVisitProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -2201,7 +3080,13 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "GtScanRequest", "HbaseRawScan", });
+ new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "HbaseColumnsToGT", });
+ internal_static_CubeVisitRequest_IntList_descriptor =
+ internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
+ internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_CubeVisitRequest_IntList_descriptor,
+ new java.lang.String[] { "Ints", });
internal_static_CubeVisitResponse_descriptor =
getDescriptor().getMessageTypes().get(1);
internal_static_CubeVisitResponse_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index e6c20a4..a4cd39d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -14,6 +14,10 @@ option optimize_for = SPEED;
message CubeVisitRequest {
required bytes gtScanRequest = 1;
required bytes hbaseRawScan = 2;
+ repeated IntList hbaseColumnsToGT = 3;
+ message IntList {
+ repeated int32 ints = 1;
+ }
}
message CubeVisitResponse {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 9ff7aa4..6d3ec4d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -51,9 +51,9 @@ import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.TsConditionExtractor;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
+import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
@@ -67,7 +67,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.google.common.collect.Ranges;
import com.google.common.collect.Sets;
-import com.google.protobuf.ByteString;
+import com.google.protobuf.HBaseZeroCopyByteString;
/**
*/
@@ -152,7 +152,7 @@ public class EndpointTupleIterator implements ITupleIterator {
//decompress
Collection<IIProtos.IIResponseInternal> shardResults = new ArrayList<>();
for (IIProtos.IIResponse input : compressedShardResults) {
- byte[] compressed = input.getBlob().toByteArray();
+ byte[] compressed = HBaseZeroCopyByteString.zeroCopyGetBytes(input.getBlob());
try {
byte[] decompressed = CompressionUtils.decompress(compressed);
shardResults.add(IIProtos.IIResponseInternal.parseFrom(decompressed));
@@ -275,20 +275,19 @@ public class EndpointTupleIterator implements ITupleIterator {
if (this.tsRange != null) {
byte[] tsRangeBytes = SerializationUtils.serialize(this.tsRange);
- builder.setTsRange(ByteString.copyFrom(tsRangeBytes));
+ builder.setTsRange(HBaseZeroCopyByteString.wrap(tsRangeBytes));
}
- builder.setType(ByteString.copyFrom(CoprocessorRowType.serialize(pushedDownRowType))) //
- .setFilter(ByteString.copyFrom(CoprocessorFilter.serialize(pushedDownFilter))) //
- .setProjector(ByteString.copyFrom(CoprocessorProjector.serialize(pushedDownProjector))) //
- .setAggregator(ByteString.copyFrom(EndpointAggregators.serialize(pushedDownAggregators)));
+ builder.setType(HBaseZeroCopyByteString.wrap(CoprocessorRowType.serialize(pushedDownRowType))) //
+ .setFilter(HBaseZeroCopyByteString.wrap(CoprocessorFilter.serialize(pushedDownFilter))) //
+ .setProjector(HBaseZeroCopyByteString.wrap(CoprocessorProjector.serialize(pushedDownProjector))) //
+ .setAggregator(HBaseZeroCopyByteString.wrap(EndpointAggregators.serialize(pushedDownAggregators)));
IIProtos.IIRequest request = builder.build();
return request;
}
- //TODO : async callback
private Collection<IIProtos.IIResponse> getResults(final IIProtos.IIRequest request, HTableInterface table) throws Throwable {
Map<byte[], IIProtos.IIResponse> results = table.coprocessorService(IIProtos.RowsService.class, null, null, new Batch.Call<IIProtos.RowsService, IIProtos.IIResponse>() {
public IIProtos.IIResponse call(IIProtos.RowsService rowsService) throws IOException {
@@ -338,10 +337,10 @@ public class EndpointTupleIterator implements ITupleIterator {
}
IIProtos.IIResponseInternal.IIRow currentRow = rows.get(index);
- byte[] columnsBytes = currentRow.getColumns().toByteArray();
+ byte[] columnsBytes = HBaseZeroCopyByteString.zeroCopyGetBytes(currentRow.getColumns());
this.tableRecord.setBytes(columnsBytes, 0, columnsBytes.length);
if (currentRow.hasMeasures()) {
- byte[] measuresBytes = currentRow.getMeasures().toByteArray();
+ byte[] measuresBytes = HBaseZeroCopyByteString.zeroCopyGetBytes(currentRow.getMeasures());
this.measureValues = pushedDownAggregators.deserializeMetricValues(measuresBytes, 0);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
index 5f8fefe..6173241 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
@@ -53,15 +53,15 @@ import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Range;
-import com.google.protobuf.ByteString;
+import com.google.protobuf.HBaseZeroCopyByteString;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
@@ -88,7 +88,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
scan.addColumn(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES);
if (request.hasTsRange()) {
- Range<Long> tsRange = (Range<Long>) SerializationUtils.deserialize(request.getTsRange().toByteArray());
+ Range<Long> tsRange = (Range<Long>) SerializationUtils.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getTsRange()));
byte[] regionStartKey = region.getStartKey();
if (!ArrayUtils.isEmpty(regionStartKey)) {
shard = BytesUtil.readUnsigned(regionStartKey, 0, IIKeyValueCodec.SHARD_LEN);
@@ -148,15 +148,15 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
innerScanner = region.getScanner(prepareScan(request, region));
- CoprocessorRowType type = CoprocessorRowType.deserialize(request.getType().toByteArray());
- CoprocessorProjector projector = CoprocessorProjector.deserialize(request.getProjector().toByteArray());
- EndpointAggregators aggregators = EndpointAggregators.deserialize(request.getAggregator().toByteArray());
- CoprocessorFilter filter = CoprocessorFilter.deserialize(request.getFilter().toByteArray());
+ CoprocessorRowType type = CoprocessorRowType.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getType()));
+ CoprocessorProjector projector = CoprocessorProjector.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getProjector()));
+ EndpointAggregators aggregators = EndpointAggregators.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getAggregator()));
+ CoprocessorFilter filter = CoprocessorFilter.deserialize(HBaseZeroCopyByteString.zeroCopyGetBytes(request.getFilter()));
//compression
IIProtos.IIResponseInternal response = getResponse(innerScanner, type, projector, aggregators, filter);
byte[] compressed = CompressionUtils.compress(response.toByteArray());
- IIProtos.IIResponse compressedR = IIProtos.IIResponse.newBuilder().setBlob(ByteString.copyFrom(compressed)).build();
+ IIProtos.IIResponse compressedR = IIProtos.IIResponse.newBuilder().setBlob(HBaseZeroCopyByteString.wrap(compressed)).build();
done.run(compressedR);
} catch (IOException ioe) {
@@ -257,7 +257,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
if (totalByteFormLen >= MEMORY_LIMIT) {
throw new RuntimeException("the query has exceeded the memory limit, please check the query");
}
- IIProtos.IIResponseInternal.IIRow.Builder rowBuilder = IIProtos.IIResponseInternal.IIRow.newBuilder().setColumns(ByteString.copyFrom(recordBuffer));
+ IIProtos.IIResponseInternal.IIRow.Builder rowBuilder = IIProtos.IIResponseInternal.IIRow.newBuilder().setColumns(HBaseZeroCopyByteString.wrap(recordBuffer));
responseBuilder.addRows(rowBuilder.build());
totalByteFormLen += byteFormLen;
}
@@ -269,9 +269,9 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
if (needAgg) {
for (Map.Entry<AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) {
AggrKey aggrKey = entry.getKey();
- IIProtos.IIResponseInternal.IIRow.Builder rowBuilder = IIProtos.IIResponseInternal.IIRow.newBuilder().setColumns(ByteString.copyFrom(aggrKey.get(), aggrKey.offset(), aggrKey.length()));
+ IIProtos.IIResponseInternal.IIRow.Builder rowBuilder = IIProtos.IIResponseInternal.IIRow.newBuilder().setColumns(HBaseZeroCopyByteString.wrap(aggrKey.get(), aggrKey.offset(), aggrKey.length()));
int length = aggregators.serializeMetricValues(entry.getValue(), buffer);
- rowBuilder.setMeasures(ByteString.copyFrom(buffer, 0, length));
+ rowBuilder.setMeasures(HBaseZeroCopyByteString.wrap(buffer, 0, length));
responseBuilder.addRows(rowBuilder.build());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index dec6de4..b29bd81 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -24,12 +24,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.commons.cli.Options;
+import org.apache.commons.math3.primes.Primes;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -47,6 +49,8 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -55,6 +59,7 @@ import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.CuboidShardUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -79,6 +84,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
CubeDesc cubeDesc = null;
String segmentName = null;
KylinConfig kylinConfig;
+ public static final boolean ENABLE_CUBOID_SHARDING = true;
@Override
public int run(String[] args) throws Exception {
@@ -92,7 +98,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
parseOptions(options, args);
Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));
- boolean statistics_enabled = Boolean.parseBoolean(getOptionValue(OPTION_STATISTICS_ENABLED));
+ boolean statsEnabled = Boolean.parseBoolean(getOptionValue(OPTION_STATISTICS_ENABLED));
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -106,9 +112,8 @@ public class CreateHTableJob extends AbstractHadoopJob {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
try {
-
byte[][] splitKeys;
- if (statistics_enabled) {
+ if (statsEnabled) {
final Map<Long, Long> cuboidSizeMap = getCubeRowCountMapFromCuboidStatistics(cubeSegment, kylinConfig, conf);
splitKeys = getSplitsFromCuboidStatistics(cuboidSizeMap, kylinConfig, cubeSegment);
} else {
@@ -203,6 +208,17 @@ public class CreateHTableJob extends AbstractHadoopJob {
return getCubeRowCountMapFromCuboidStatistics(counterMap, samplingPercentage);
}
+ //one region for one shard
+ private static byte[][] getSplitsByRegionCount(int regionCount) {
+ byte[][] result = new byte[regionCount - 1][];
+ for (int i = 1; i < regionCount; ++i) {
+ byte[] split = new byte[Bytes.SIZEOF_SHORT];
+ BytesUtil.writeUnsigned(i, split, 0, Bytes.SIZEOF_SHORT);
+ result[i - 1] = split;
+ }
+ return result;
+ }
+
public static Map<Long, Long> getCubeRowCountMapFromCuboidStatistics(Map<Long, HyperLogLogPlusCounter> counterMap, final int samplingPercentage) throws IOException {
Preconditions.checkArgument(samplingPercentage > 0);
return Maps.transformValues(counterMap, new Function<HyperLogLogPlusCounter, Long>() {
@@ -233,58 +249,109 @@ public class CreateHTableJob extends AbstractHadoopJob {
logger.info("Cube capacity " + cubeCapacity.toString() + ", chosen cut for HTable is " + cut + "GB");
- long totalSizeInM = 0;
+ double totalSizeInM = 0;
List<Long> allCuboids = Lists.newArrayList();
allCuboids.addAll(cubeRowCountMap.keySet());
Collections.sort(allCuboids);
- Map<Long, Long> cubeSizeMap = Maps.transformEntries(cubeRowCountMap, new Maps.EntryTransformer<Long, Long, Long>() {
- @Override
- public Long transformEntry(@Nullable Long key, @Nullable Long value) {
- return estimateCuboidStorageSize(cubeDesc, key, value, baseCuboidId, rowkeyColumnSize);
- }
- });
- for (Long cuboidSize : cubeSizeMap.values()) {
+ Map<Long, Double> cubeSizeMap = Maps.newHashMap();
+ for (Map.Entry<Long, Long> entry : cubeRowCountMap.entrySet()) {
+ cubeSizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeDesc, entry.getKey(), entry.getValue(), baseCuboidId, rowkeyColumnSize));
+ }
+
+ for (Double cuboidSize : cubeSizeMap.values()) {
totalSizeInM += cuboidSize;
}
- int nRegion = Math.round((float) totalSizeInM / (cut * 1024L));
+ int nRegion = Math.round((float) (totalSizeInM / (cut * 1024L)));
nRegion = Math.max(kylinConfig.getHBaseRegionCountMin(), nRegion);
nRegion = Math.min(kylinConfig.getHBaseRegionCountMax(), nRegion);
- int mbPerRegion = (int) (totalSizeInM / (nRegion));
+ if (ENABLE_CUBOID_SHARDING) {//&& (nRegion > 1)) {
+ //use prime nRegions to help random sharding
+ int original = nRegion;
+ nRegion = Primes.nextPrime(nRegion);//return 2 for input 1
+
+ if (nRegion > Short.MAX_VALUE) {
+ logger.info("Too many regions! reduce to " + Short.MAX_VALUE);
+ nRegion = Short.MAX_VALUE;
+ }
+
+ if (nRegion != original) {
+ logger.info("Region count is adjusted from " + original + " to " + nRegion + " to help random sharding");
+ }
+ }
+
+ int mbPerRegion = (int) (totalSizeInM / nRegion);
mbPerRegion = Math.max(1, mbPerRegion);
logger.info("Total size " + totalSizeInM + "M (estimated)");
- logger.info(nRegion + " regions (estimated)");
- logger.info(mbPerRegion + " MB per region (estimated)");
-
- List<Long> regionSplit = Lists.newArrayList();
-
- long size = 0;
- int regionIndex = 0;
- int cuboidCount = 0;
- for (int i = 0; i < allCuboids.size(); i++) {
- long cuboidId = allCuboids.get(i);
- if (size >= mbPerRegion || (size + cubeSizeMap.get(cuboidId)) >= mbPerRegion * 1.2) {
- // if the size already bigger than threshold, or it will exceed by 20%, cut for next region
- regionSplit.add(cuboidId);
- logger.info("Region " + regionIndex + " will be " + size + " MB, contains cuboids < " + cuboidId + " (" + cuboidCount + ") cuboids");
- size = 0;
- cuboidCount = 0;
- regionIndex++;
+ logger.info("Expecting " + nRegion + " regions.");
+ logger.info("Expecting " + mbPerRegion + " MB per region.");
+
+ if (ENABLE_CUBOID_SHARDING) {
+ //each cuboid will be split into different number of shards
+ HashMap<Long, Short> cuboidShards = Maps.newHashMap();
+ double[] regionSizes = new double[nRegion];
+ for (long cuboidId : allCuboids) {
+ double estimatedSize = cubeSizeMap.get(cuboidId);
+ double magic = 23;
+ int shardNum = (int) (estimatedSize * magic / mbPerRegion + 1);
+ if (shardNum < 1) {
+ shardNum = 1;
+ }
+
+ if (shardNum > nRegion) {
+ logger.info(String.format("Cuboid %d 's estimated size %.2f MB will generate %d regions, reduce to %d", cuboidId, estimatedSize, shardNum, nRegion));
+ shardNum = nRegion;
+ } else {
+ logger.info(String.format("Cuboid %d 's estimated size %.2f MB will generate %d regions", cuboidId, estimatedSize, shardNum));
+ }
+
+ cuboidShards.put(cuboidId, (short) shardNum);
+ short startShard = ShardingHash.getShard(cuboidId, nRegion);
+ for (short i = startShard; i < startShard + shardNum; ++i) {
+ short j = (short) (i % nRegion);
+ regionSizes[j] = regionSizes[j] + estimatedSize / shardNum;
+ }
}
- size += cubeSizeMap.get(cuboidId);
- cuboidCount++;
- }
- byte[][] result = new byte[regionSplit.size()][];
- for (int i = 0; i < regionSplit.size(); i++) {
- result[i] = Bytes.toBytes(regionSplit.get(i));
- }
+ for (int i = 0; i < nRegion; ++i) {
+ logger.info(String.format("Region %d's estimated size is %.2f MB, accounting for %.2f percent", i, regionSizes[i], 100.0 * regionSizes[i] / totalSizeInM));
+ }
- return result;
+ CuboidShardUtil.saveCuboidShards(cubeSegment, cuboidShards, nRegion);
+
+ return getSplitsByRegionCount(nRegion);
+
+ } else {
+ List<Long> regionSplit = Lists.newArrayList();
+
+ long size = 0;
+ int regionIndex = 0;
+ int cuboidCount = 0;
+ for (int i = 0; i < allCuboids.size(); i++) {
+ long cuboidId = allCuboids.get(i);
+ if (size >= mbPerRegion || (size + cubeSizeMap.get(cuboidId)) >= mbPerRegion * 1.2) {
+ // if the size already bigger than threshold, or it will exceed by 20%, cut for next region
+ regionSplit.add(cuboidId);
+ logger.info("Region " + regionIndex + " will be " + size + " MB, contains cuboids < " + cuboidId + " (" + cuboidCount + ") cuboids");
+ size = 0;
+ cuboidCount = 0;
+ regionIndex++;
+ }
+ size += cubeSizeMap.get(cuboidId);
+ cuboidCount++;
+ }
+
+ byte[][] result = new byte[regionSplit.size()][];
+ for (int i = 0; i < regionSplit.size(); i++) {
+ result[i] = Bytes.toBytes(regionSplit.get(i));
+ }
+
+ return result;
+ }
}
/**
@@ -295,9 +362,9 @@ public class CreateHTableJob extends AbstractHadoopJob {
* @param rowCount
* @return the cuboid size in M bytes
*/
- private static long estimateCuboidStorageSize(CubeDesc cubeDesc, long cuboidId, long rowCount, long baseCuboidId, List<Integer> rowKeyColumnLength) {
+ private static double estimateCuboidStorageSize(CubeDesc cubeDesc, long cuboidId, long rowCount, long baseCuboidId, List<Integer> rowKeyColumnLength) {
- int bytesLength = RowConstants.ROWKEY_CUBOIDID_LEN;
+ int bytesLength = RowConstants.ROWKEY_HEADER_LEN;
long mask = Long.highestOneBit(baseCuboidId);
long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
@@ -321,9 +388,9 @@ public class CreateHTableJob extends AbstractHadoopJob {
}
bytesLength += space;
- logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + bytesLength + " bytes.");
- logger.info("Cuboid " + cuboidId + " total size is " + (bytesLength * rowCount / (1024L * 1024L)) + "M.");
- return bytesLength * rowCount / (1024L * 1024L);
+ double ret = 1.0 * bytesLength * rowCount / (1024L * 1024L);
+ logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + bytesLength + " bytes." + " Total size is " + ret + "M.");
+ return ret;
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index e698b27..d857fb1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -43,9 +43,13 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.ShardingHash;
+import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
+import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
@@ -68,12 +72,14 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
private final HTableInterface hTable;
private final ByteBuffer byteBuffer;
private final CubeDesc cubeDesc;
+ private final CubeSegment cubeSegment;
private final Object[] measureValues;
private List<Put> puts = Lists.newArrayList();
- public HBaseCuboidWriter(CubeDesc cubeDesc, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
this.keyValueCreators = Lists.newArrayList();
- this.cubeDesc = cubeDesc;
+ this.cubeSegment = segment;
+ this.cubeDesc = cubeSegment.getCubeDesc();
for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
@@ -81,7 +87,7 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
}
this.nColumns = keyValueCreators.size();
this.hTable = hTable;
- this.byteBuffer = ByteBuffer.allocate(1 << 20);
+ this.byteBuffer = ByteBuffer.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
this.measureValues = new Object[cubeDesc.getMeasures().size()];
}
@@ -93,12 +99,22 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
private ByteBuffer createKey(Long cuboidId, GTRecord record) {
byteBuffer.clear();
- byteBuffer.put(Bytes.toBytes(cuboidId));
+ byteBuffer.put(Bytes.toBytes((short) 0), 0, RowConstants.ROWKEY_SHARDID_LEN);//occupy space first
+ byteBuffer.put(Bytes.toBytes(cuboidId), 0, RowConstants.ROWKEY_CUBOIDID_LEN);
final int cardinality = BitSet.valueOf(new long[] { cuboidId }).cardinality();
for (int i = 0; i < cardinality; i++) {
final ByteArray byteArray = record.get(i);
byteBuffer.put(byteArray.array(), byteArray.offset(), byteArray.length());
}
+
+ //fill shard
+ short cuboidShardNum = cubeSegment.getCuboidShardNum(cuboidId);
+ short shardOffset = ShardingHash.getShard(byteBuffer.array(), //
+ RowConstants.ROWKEY_HEADER_LEN, byteBuffer.position() - RowConstants.ROWKEY_HEADER_LEN, cuboidShardNum);
+ Short cuboidShardBase = cubeSegment.getCuboidBaseShard(cuboidId);
+ short finalShard = ShardingHash.normalize(cuboidShardBase, shardOffset, cubeSegment.getTotalShards());
+ BytesUtil.writeShort(finalShard, byteBuffer.array(), 0, RowConstants.ROWKEY_SHARDID_LEN);
+
return byteBuffer;
}
@@ -108,7 +124,7 @@ public final class HBaseCuboidWriter implements ICuboidWriter {
final Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
final int nDims = cuboid.getColumns().size();
final ImmutableBitSet bitSet = new ImmutableBitSet(nDims, nDims + cubeDesc.getMeasures().size());
-
+
for (int i = 0; i < nColumns; i++) {
final Object[] values = record.getValues(bitSet, measureValues);
final KeyValue keyValue = keyValueCreators.get(i).create(key.array(), 0, key.position(), values);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index b4c0d61..7b4e1a4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -34,7 +34,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.steps.FactDistinctColumnsReducer;
+import org.apache.kylin.engine.mr.common.CuboidStatsUtil;
import org.apache.kylin.engine.streaming.IStreamingOutput;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -53,7 +53,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
CubeSegment cubeSegment = (CubeSegment) buildable;
final HTableInterface hTable;
hTable = createHTable(cubeSegment);
- return new HBaseCuboidWriter(cubeSegment.getCubeDesc(), hTable);
+ return new HBaseCuboidWriter(cubeSegment, hTable);
} catch (IOException e) {
throw new RuntimeException("failed to get ICuboidWriter", e);
}
@@ -66,7 +66,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
final Configuration conf = HadoopUtil.getCurrentConfiguration();
final Path outputPath = new Path("file://" + BatchConstants.CFG_STATISTICS_LOCAL_DIR + UUID.randomUUID().toString());
- FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
+ CuboidStatsUtil.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
FSDataInputStream inputStream = null;
try {
inputStream = FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index df42560..a4a8a35 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -56,8 +56,16 @@ public class MergeGCStep extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ try {
+ logger.info("Sleep one minute before deleting the Htables");
+ Thread.sleep(60000);
+ } catch (InterruptedException e) {
+ logger.warn("Thread interrupted");
+ }
+
+ logger.info("Start doing merge gc work");
+
StringBuffer output = new StringBuffer();
-
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fabdd5cd/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index fadab1a..a003d6a 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -34,7 +34,7 @@ import org.apache.kylin.common.util.ClassUtil;
* It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
*/
public class SandboxMetastoreCLI {
-
+
private static final Log logger = LogFactory.getLog(SandboxMetastoreCLI.class);
public static void main(String[] args) throws Exception {