You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by je...@apache.org on 2015/02/07 03:08:32 UTC
[3/3] hbase git commit: HBASE-11567 Write bulk load COMMIT events to
WAL (Alex Newman, Jeffrey Zhong)
HBASE-11567 Write bulk load COMMIT events to WAL (Alex Newman, Jeffrey Zhong)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3f442773
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3f442773
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3f442773
Branch: refs/heads/master
Commit: 3f4427739d9ff698d39f2687f11f65967c67340d
Parents: 7f4146b
Author: Jeffrey Zhong <je...@apache.org>
Authored: Wed Feb 4 15:52:01 2015 -0800
Committer: Jeffrey Zhong <je...@apache.org>
Committed: Fri Feb 6 18:07:30 2015 -0800
----------------------------------------------------------------------
.../hadoop/hbase/protobuf/ProtobufUtil.java | 35 +-
.../hbase/protobuf/generated/FilterProtos.java | 6 +-
.../generated/VisibilityLabelsProtos.java | 6 +-
.../hbase/protobuf/generated/WALProtos.java | 3441 ++++++++++++------
hbase-protocol/src/main/protobuf/WAL.proto | 23 +-
.../hadoop/hbase/regionserver/HRegion.java | 95 +-
.../hadoop/hbase/regionserver/wal/WALEdit.java | 35 +-
.../hadoop/hbase/regionserver/wal/WALUtil.java | 39 +
.../hadoop/hbase/regionserver/TestBulkLoad.java | 312 ++
.../hadoop/hbase/regionserver/TestHRegion.java | 2 +-
.../regionserver/TestHRegionServerBulkLoad.java | 46 +-
.../wal/TestWALActionsListener.java | 2 +-
pom.xml | 17 +
13 files changed, 2869 insertions(+), 1190 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3f442773/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 9c383f8..caae1bb 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -125,6 +125,8 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.FlushDescriptor.FlushAction;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor;
import org.apache.hadoop.hbase.protobuf.generated.WALProtos.RegionEventDescriptor.EventType;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hadoop.hbase.protobuf.generated.WALProtos.StoreDescriptor;
import org.apache.hadoop.hbase.quotas.QuotaScope;
import org.apache.hadoop.hbase.quotas.QuotaType;
import org.apache.hadoop.hbase.quotas.ThrottleType;
@@ -2608,8 +2610,7 @@ public final class ProtobufUtil {
.setServer(toServerName(server));
for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
- RegionEventDescriptor.StoreDescriptor.Builder builder
- = RegionEventDescriptor.StoreDescriptor.newBuilder()
+ StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
.setFamilyName(ByteStringer.wrap(entry.getKey()))
.setStoreHomeDir(Bytes.toString(entry.getKey()));
for (Path path : entry.getValue()) {
@@ -2963,4 +2964,34 @@ public final class ProtobufUtil {
.setScope(toProtoQuotaScope(scope))
.build();
}
+
+ /**
+ * Generates a marker for the WAL so that we propagate the notion of a bulk region load
+ * throughout the WAL.
+ *
+ * @param tableName The tableName into which the bulk load is being imported into.
+ * @param encodedRegionName Encoded region name of the region which is being bulk loaded.
+ * @param storeFiles A set of store files of a column family are bulk loaded.
+ * @param bulkloadSeqId sequence ID (by a force flush) used to create bulk load hfile
+ * name
+ * @return The WAL log marker for bulk loads.
+ */
+ public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName,
+ ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles, long bulkloadSeqId) {
+ BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
+ .setTableName(ProtobufUtil.toProtoTableName(tableName))
+ .setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
+
+ for (Map.Entry<byte[], List<Path>> entry : storeFiles.entrySet()) {
+ WALProtos.StoreDescriptor.Builder builder = StoreDescriptor.newBuilder()
+ .setFamilyName(ByteStringer.wrap(entry.getKey()))
+ .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to region
+ for (Path path : entry.getValue()) {
+ builder.addStoreFile(path.getName());
+ }
+ desc.addStores(builder);
+ }
+
+ return desc.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3f442773/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/FilterProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/FilterProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/FilterProtos.java
index 3fb466d..af1f33d 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/FilterProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/FilterProtos.java
@@ -16611,7 +16611,7 @@ public final class FilterProtos {
/**
* <code>repeated .RowRange row_range_list = 1;</code>
*/
- java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRangeOrBuilder>
+ java.util.List<? extends org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRangeOrBuilder>
getRowRangeListOrBuilderList();
/**
* <code>repeated .RowRange row_range_list = 1;</code>
@@ -17270,12 +17270,12 @@ public final class FilterProtos {
/**
* <code>repeated .RowRange row_range_list = 1;</code>
*/
- public java.util.List<org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRange.Builder>
+ public java.util.List<org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRange.Builder>
getRowRangeListBuilderList() {
return getRowRangeListFieldBuilder().getBuilderList();
}
private com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRange, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRange.Builder, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRangeOrBuilder>
+ org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRange, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRange.Builder, org.apache.hadoop.hbase.protobuf.generated.FilterProtos.RowRangeOrBuilder>
getRowRangeListFieldBuilder() {
if (rowRangeListBuilder_ == null) {
rowRangeListBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
http://git-wip-us.apache.org/repos/asf/hbase/blob/3f442773/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/VisibilityLabelsProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/VisibilityLabelsProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/VisibilityLabelsProtos.java
index 294772e..70593b0 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/VisibilityLabelsProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/VisibilityLabelsProtos.java
@@ -5092,7 +5092,7 @@ public final class VisibilityLabelsProtos {
if (ref instanceof java.lang.String) {
return (java.lang.String) ref;
} else {
- com.google.protobuf.ByteString bs =
+ com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
java.lang.String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -5108,7 +5108,7 @@ public final class VisibilityLabelsProtos {
getRegexBytes() {
java.lang.Object ref = regex_;
if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(java.lang.String) ref);
regex_ = b;
@@ -5414,7 +5414,7 @@ public final class VisibilityLabelsProtos {
getRegexBytes() {
java.lang.Object ref = regex_;
if (ref instanceof String) {
- com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(java.lang.String) ref);
regex_ = b;