You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2022/01/28 20:58:55 UTC
[hbase] branch branch-2.4 updated (21de18e -> 590e38e)
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a change to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 21de18e HBASE-26679 Wait on the future returned by FanOutOneBlockAsyncDFSOutput.flush would stuck (#4077)
new 70018b0 HBASE-25918 Upgrade hbase-thirdparty dependency to 3.5.1 (#3317)
new 590e38e HBASE-26713 Default to LATEST_TIMESTAMP if no timestamp sent along on Increment/Append (#4075)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 15 ++-
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 15 ++-
.../hbase/shaded/protobuf/TestProtobufUtil.java | 103 +++++++++++++++------
.../hadoop/hbase/protobuf/TestProtobufUtil.java | 103 ++++++++++++++-------
pom.xml | 2 +-
5 files changed, 165 insertions(+), 73 deletions(-)
[hbase] 02/02: HBASE-26713 Default to LATEST_TIMESTAMP if no timestamp sent along on Increment/Append (#4075)
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 590e38e8e479011bfb49b6ec00cc930645f84a5c
Author: Bryan Beaudreault <bb...@hubspot.com>
AuthorDate: Fri Jan 28 15:55:51 2022 -0500
HBASE-26713 Default to LATEST_TIMESTAMP if no timestamp sent along on Increment/Append (#4075)
Signed-off-by: Andrew Purtell <ap...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../apache/hadoop/hbase/protobuf/ProtobufUtil.java | 15 ++-
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 15 ++-
.../hbase/shaded/protobuf/TestProtobufUtil.java | 103 +++++++++++++++------
.../hadoop/hbase/protobuf/TestProtobufUtil.java | 103 ++++++++++++++-------
4 files changed, 164 insertions(+), 72 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index dc7f7d2..9c86509 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -625,10 +625,7 @@ public final class ProtobufUtil {
if (qv.hasQualifier()) {
qualifier = qv.getQualifier().toByteArray();
}
- long ts = HConstants.LATEST_TIMESTAMP;
- if (qv.hasTimestamp()) {
- ts = qv.getTimestamp();
- }
+ long ts = cellTimestampOrLatest(qv);
if (deleteType == DeleteType.DELETE_ONE_VERSION) {
delete.addColumn(family, qualifier, ts);
} else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
@@ -692,7 +689,7 @@ public final class ProtobufUtil {
if (qv.hasTags()) {
tags = qv.getTags().toByteArray();
}
- consumer.accept(mutation, CellUtil.createCell(mutation.getRow(), family, qualifier, qv.getTimestamp(),
+ consumer.accept(mutation, CellUtil.createCell(mutation.getRow(), family, qualifier, cellTimestampOrLatest(qv),
KeyValue.Type.Put, value, tags));
}
}
@@ -704,6 +701,14 @@ public final class ProtobufUtil {
return mutation;
}
+ private static long cellTimestampOrLatest(QualifierValue cell) {
+ if (cell.hasTimestamp()) {
+ return cell.getTimestamp();
+ } else {
+ return HConstants.LATEST_TIMESTAMP;
+ }
+ }
+
/**
* Convert a protocol buffer Mutate to an Append
* @param cellScanner
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index c2544f6..25a2f10 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -785,10 +785,7 @@ public final class ProtobufUtil {
if (qv.hasQualifier()) {
qualifier = qv.getQualifier().toByteArray();
}
- long ts = HConstants.LATEST_TIMESTAMP;
- if (qv.hasTimestamp()) {
- ts = qv.getTimestamp();
- }
+ long ts = cellTimestampOrLatest(qv);
if (deleteType == DeleteType.DELETE_ONE_VERSION) {
delete.addColumn(family, qualifier, ts);
} else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
@@ -855,7 +852,7 @@ public final class ProtobufUtil {
.setRow(mutation.getRow())
.setFamily(family)
.setQualifier(qualifier)
- .setTimestamp(qv.getTimestamp())
+ .setTimestamp(cellTimestampOrLatest(qv))
.setType(KeyValue.Type.Put.getCode())
.setValue(value)
.setTags(tags)
@@ -870,6 +867,14 @@ public final class ProtobufUtil {
return mutation;
}
+ private static long cellTimestampOrLatest(QualifierValue cell) {
+ if (cell.hasTimestamp()) {
+ return cell.getTimestamp();
+ } else {
+ return HConstants.LATEST_TIMESTAMP;
+ }
+ }
+
/**
* Convert a protocol buffer Mutate to an Append
* @param cellScanner
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java
index c47150b..317dff9 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.ExtendedCellBuilder;
import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.Tag;
@@ -294,7 +295,22 @@ public class TestProtobufUtil {
*/
@Test
public void testIncrement() throws IOException {
- long timeStamp = 111111;
+
+ MutationProto proto = getIncrementMutation(111111L);
+ // default fields
+ assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
+
+ // set the default value for equal comparison
+ MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
+ mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
+
+ Increment increment = ProtobufUtil.toIncrement(proto, null);
+ mutateBuilder.setTimestamp(increment.getTimestamp());
+ mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
+ assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
+ }
+
+ private MutationProto getIncrementMutation(Long timestamp) {
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
mutateBuilder.setMutateType(MutationProto.MutationType.INCREMENT);
@@ -303,66 +319,93 @@ public class TestProtobufUtil {
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(11L)));
- qualifierBuilder.setTimestamp(timeStamp);
+
+ if (timestamp != null) {
+ qualifierBuilder.setTimestamp(timestamp);
+ }
+
valueBuilder.addQualifierValue(qualifierBuilder.build());
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(22L)));
valueBuilder.addQualifierValue(qualifierBuilder.build());
mutateBuilder.addColumnValue(valueBuilder.build());
- MutationProto proto = mutateBuilder.build();
+ return mutateBuilder.build();
+ }
+
+ /**
+ * Older clients may not send along a timestamp in the MutationProto. Check that we
+ * default correctly.
+ */
+ @Test
+ public void testIncrementNoTimestamp() throws IOException {
+ MutationProto mutation = getIncrementMutation(null);
+ Increment increment = ProtobufUtil.toIncrement(mutation, null);
+ assertEquals(HConstants.LATEST_TIMESTAMP, increment.getTimestamp());
+ increment.getFamilyCellMap().values()
+ .forEach(cells ->
+ cells.forEach(cell ->
+ assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
+ }
+
+ /**
+ * Test Append Mutate conversions.
+ *
+ * @throws IOException if converting to an {@link Append} fails
+ */
+ @Test
+ public void testAppend() throws IOException {
+ MutationProto proto = getAppendMutation(111111L);
// default fields
assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
// set the default value for equal comparison
- mutateBuilder = MutationProto.newBuilder(proto);
+ MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
- Increment increment = ProtobufUtil.toIncrement(proto, null);
- mutateBuilder.setTimestamp(increment.getTimestamp());
- mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
- assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
+ Append append = ProtobufUtil.toAppend(proto, null);
+
+ // append always use the latest timestamp,
+ // reset the timestamp to the original mutate
+ mutateBuilder.setTimestamp(append.getTimestamp());
+ mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
+ assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
}
/**
- * Test Append Mutate conversions.
- *
- * @throws IOException if converting to an {@link Append} fails
+ * Older clients may not send along a timestamp in the MutationProto. Check that we
+ * default correctly.
*/
@Test
- public void testAppend() throws IOException {
- long timeStamp = 111111;
+ public void testAppendNoTimestamp() throws IOException {
+ MutationProto mutation = getAppendMutation(null);
+ Append append = ProtobufUtil.toAppend(mutation, null);
+ assertEquals(HConstants.LATEST_TIMESTAMP, append.getTimestamp());
+ append.getFamilyCellMap().values().forEach(cells -> cells.forEach(cell -> assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
+ }
+
+ private MutationProto getAppendMutation(Long timestamp) {
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
mutateBuilder.setMutateType(MutationType.APPEND);
- mutateBuilder.setTimestamp(timeStamp);
+ if (timestamp != null) {
+ mutateBuilder.setTimestamp(timestamp);
+ }
ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
- qualifierBuilder.setTimestamp(timeStamp);
+ if (timestamp != null) {
+ qualifierBuilder.setTimestamp(timestamp);
+ }
valueBuilder.addQualifierValue(qualifierBuilder.build());
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
valueBuilder.addQualifierValue(qualifierBuilder.build());
mutateBuilder.addColumnValue(valueBuilder.build());
- MutationProto proto = mutateBuilder.build();
- // default fields
- assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
-
- // set the default value for equal comparison
- mutateBuilder = MutationProto.newBuilder(proto);
- mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
-
- Append append = ProtobufUtil.toAppend(proto, null);
-
- // append always use the latest timestamp,
- // reset the timestamp to the original mutate
- mutateBuilder.setTimestamp(append.getTimestamp());
- mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
- assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
+ return mutateBuilder.build();
}
private static ProcedureProtos.Procedure.Builder createProcedureBuilder(long procId) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
index 69e656f..2d8d8e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
@@ -122,38 +122,57 @@ public class TestProtobufUtil {
*/
@Test
public void testAppend() throws IOException {
- long timeStamp = 111111;
+ MutationProto proto = getAppendMutation(111111L);
+ // default fields
+ assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
+
+ // set the default value for equal comparison
+ MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
+ mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
+
+ Append append = ProtobufUtil.toAppend(proto, null);
+
+ // append always use the latest timestamp,
+ // reset the timestamp to the original mutate
+ mutateBuilder.setTimestamp(append.getTimestamp());
+ mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
+ assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
+ }
+
+ /**
+ * Older clients may not send along a timestamp in the MutationProto. Check that we
+ * default correctly.
+ */
+ @Test
+ public void testAppendNoTimestamp() throws IOException {
+ MutationProto mutation = getAppendMutation(null);
+ Append append = ProtobufUtil.toAppend(mutation, null);
+ assertEquals(HConstants.LATEST_TIMESTAMP, append.getTimestamp());
+ append.getFamilyCellMap().values().forEach(cells -> cells.forEach(cell -> assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
+ }
+
+ private MutationProto getAppendMutation(Long timestamp) {
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
mutateBuilder.setMutateType(MutationType.APPEND);
- mutateBuilder.setTimestamp(timeStamp);
+ if (timestamp != null) {
+ mutateBuilder.setTimestamp(timestamp);
+ }
ColumnValue.Builder valueBuilder = ColumnValue.newBuilder();
valueBuilder.setFamily(ByteString.copyFromUtf8("f1"));
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
qualifierBuilder.setValue(ByteString.copyFromUtf8("v1"));
- qualifierBuilder.setTimestamp(timeStamp);
+ if (timestamp != null) {
+ qualifierBuilder.setTimestamp(timestamp);
+ }
valueBuilder.addQualifierValue(qualifierBuilder.build());
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
qualifierBuilder.setValue(ByteString.copyFromUtf8("v2"));
valueBuilder.addQualifierValue(qualifierBuilder.build());
mutateBuilder.addColumnValue(valueBuilder.build());
- MutationProto proto = mutateBuilder.build();
- // default fields
- assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
-
- // set the default value for equal comparison
- mutateBuilder = MutationProto.newBuilder(proto);
- mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
-
- Append append = ProtobufUtil.toAppend(proto, null);
-
- // append always use the latest timestamp,
- // reset the timestamp to the original mutate
- mutateBuilder.setTimestamp(append.getTimestamp());
- mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(append.getTimeRange()));
- assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.APPEND, append));
+ return mutateBuilder.build();
}
/**
@@ -210,7 +229,36 @@ public class TestProtobufUtil {
*/
@Test
public void testIncrement() throws IOException {
- long timeStamp = 111111;
+ MutationProto proto = getIncrementMutation(111111L);
+ // default fields
+ assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
+
+ // set the default value for equal comparison
+ MutationProto.Builder mutateBuilder = MutationProto.newBuilder(proto);
+ mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
+
+ Increment increment = ProtobufUtil.toIncrement(proto, null);
+ mutateBuilder.setTimestamp(increment.getTimestamp());
+ mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
+ assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
+ }
+
+ /**
+ * Older clients may not send along a timestamp in the MutationProto. Check that we
+ * default correctly.
+ */
+ @Test
+ public void testIncrementNoTimestamp() throws IOException {
+ MutationProto mutation = getIncrementMutation(null);
+ Increment increment = ProtobufUtil.toIncrement(mutation, null);
+ assertEquals(HConstants.LATEST_TIMESTAMP, increment.getTimestamp());
+ increment.getFamilyCellMap().values()
+ .forEach(cells ->
+ cells.forEach(cell ->
+ assertEquals(HConstants.LATEST_TIMESTAMP, cell.getTimestamp())));
+ }
+
+ private MutationProto getIncrementMutation(Long timestamp) {
MutationProto.Builder mutateBuilder = MutationProto.newBuilder();
mutateBuilder.setRow(ByteString.copyFromUtf8("row"));
mutateBuilder.setMutateType(MutationType.INCREMENT);
@@ -219,25 +267,16 @@ public class TestProtobufUtil {
QualifierValue.Builder qualifierBuilder = QualifierValue.newBuilder();
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c1"));
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(11L)));
- qualifierBuilder.setTimestamp(timeStamp);
+ if (timestamp != null) {
+ qualifierBuilder.setTimestamp(timestamp);
+ }
valueBuilder.addQualifierValue(qualifierBuilder.build());
qualifierBuilder.setQualifier(ByteString.copyFromUtf8("c2"));
qualifierBuilder.setValue(ByteString.copyFrom(Bytes.toBytes(22L)));
valueBuilder.addQualifierValue(qualifierBuilder.build());
mutateBuilder.addColumnValue(valueBuilder.build());
- MutationProto proto = mutateBuilder.build();
- // default fields
- assertEquals(MutationProto.Durability.USE_DEFAULT, proto.getDurability());
-
- // set the default value for equal comparison
- mutateBuilder = MutationProto.newBuilder(proto);
- mutateBuilder.setDurability(MutationProto.Durability.USE_DEFAULT);
-
- Increment increment = ProtobufUtil.toIncrement(proto, null);
- mutateBuilder.setTimestamp(increment.getTimestamp());
- mutateBuilder.setTimeRange(ProtobufUtil.toTimeRange(increment.getTimeRange()));
- assertEquals(mutateBuilder.build(), ProtobufUtil.toMutation(MutationType.INCREMENT, increment));
+ return mutateBuilder.build();
}
/**
[hbase] 01/02: HBASE-25918 Upgrade hbase-thirdparty dependency to 3.5.1 (#3317)
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 70018b09187f2c14c6bd04af324b56af97c48842
Author: Pankaj <pa...@apache.org>
AuthorDate: Tue Jun 8 07:07:46 2021 +0530
HBASE-25918 Upgrade hbase-thirdparty dependency to 3.5.1 (#3317)
Signed-off-by: Duo Zhang <zh...@apache.org>
(cherry picked from commit be14605898a5c8162d98760479ef01dd76d3d46d)
Reapplied. See HBASE-26717
Signed-off-by: Andrew Purtell <ap...@apache.org>
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 7fa760a..6650fad 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1517,7 +1517,7 @@
<surefire.version>3.0.0-M4</surefire.version>
<wagon.ssh.version>2.12</wagon.ssh.version>
<xml.maven.version>1.0.1</xml.maven.version>
- <hbase-thirdparty.version>3.4.1</hbase-thirdparty.version>
+ <hbase-thirdparty.version>3.5.1</hbase-thirdparty.version>
<!-- Intraproject jar naming properties -->
<!-- TODO this is pretty ugly, but works for the moment.
Modules are pretty heavy-weight things, so doing this work isn't too bad. -->