You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2020/12/03 15:50:06 UTC
[hbase] branch branch-2.4 updated: Revert "HBASE-25246
Backup/Restore hbase cell tags"
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 75421af Revert "HBASE-25246 Backup/Restore hbase cell tags"
75421af is described below
commit 75421af689f8cd2f826c41611822ec02fa6fa3a3
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Thu Dec 3 21:18:03 2020 +0530
Revert "HBASE-25246 Backup/Restore hbase cell tags"
This reverts commit 34721c42ec54b996444b7ea71db7955519771532.
---
.../hadoop/hbase/shaded/protobuf/ProtobufUtil.java | 24 ++--
.../hbase/shaded/protobuf/TestProtobufUtil.java | 44 ------
.../org/apache/hadoop/hbase/mapreduce/Import.java | 5 +-
.../hadoop/hbase/mapreduce/TestImportExport.java | 159 ---------------------
4 files changed, 10 insertions(+), 222 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index 5893de8..33feb29 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -1971,8 +1971,7 @@ public final class ProtobufUtil {
kvbuilder.setTimestamp(kv.getTimestamp());
kvbuilder.setValue(wrap(((ByteBufferExtendedCell) kv).getValueByteBuffer(),
((ByteBufferExtendedCell) kv).getValuePosition(), kv.getValueLength()));
- kvbuilder.setTags(wrap(((ByteBufferExtendedCell) kv).getTagsByteBuffer(),
- ((ByteBufferExtendedCell) kv).getTagsPosition(), kv.getTagsLength()));
+ // TODO : Once tags become first class then we may have to set tags to kvbuilder.
} else {
kvbuilder.setRow(
UnsafeByteOperations.unsafeWrap(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()));
@@ -1984,8 +1983,6 @@ public final class ProtobufUtil {
kvbuilder.setTimestamp(kv.getTimestamp());
kvbuilder.setValue(UnsafeByteOperations.unsafeWrap(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
- kvbuilder.setTags(UnsafeByteOperations.unsafeWrap(kv.getTagsArray(), kv.getTagsOffset(),
- kv.getTagsLength()));
}
return kvbuilder.build();
}
@@ -1998,17 +1995,14 @@ public final class ProtobufUtil {
}
public static Cell toCell(ExtendedCellBuilder cellBuilder, final CellProtos.Cell cell) {
- ExtendedCellBuilder builder = cellBuilder.clear()
- .setRow(cell.getRow().toByteArray())
- .setFamily(cell.getFamily().toByteArray())
- .setQualifier(cell.getQualifier().toByteArray())
- .setTimestamp(cell.getTimestamp())
- .setType((byte) cell.getCellType().getNumber())
- .setValue(cell.getValue().toByteArray());
- if (cell.hasTags()) {
- builder.setTags(cell.getTags().toByteArray());
- }
- return builder.build();
+ return cellBuilder.clear()
+ .setRow(cell.getRow().toByteArray())
+ .setFamily(cell.getFamily().toByteArray())
+ .setQualifier(cell.getQualifier().toByteArray())
+ .setTimestamp(cell.getTimestamp())
+ .setType((byte) cell.getCellType().getNumber())
+ .setValue(cell.getValue().toByteArray())
+ .build();
}
public static HBaseProtos.NamespaceDescriptor toProtoNamespaceDescriptor(NamespaceDescriptor ns) {
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java
index 791beb7..7d6eda8 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/shaded/protobuf/TestProtobufUtil.java
@@ -18,30 +18,22 @@
package org.apache.hadoop.hbase.shaded.protobuf;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellComparatorImpl;
-import org.apache.hadoop.hbase.ExtendedCellBuilder;
import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.PrivateCellUtil;
-import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -487,40 +479,4 @@ public class TestProtobufUtil {
+ "\"sharedLockCount\":0"
+ "}]", lockJson);
}
-
- /**
- * Test {@link ProtobufUtil#toCell(Cell)} and
- * {@link ProtobufUtil#toCell(ExtendedCellBuilder, CellProtos.Cell)} conversion
- * methods when it contains tags.
- */
- @Test
- public void testCellConversionWithTags() {
- String tagStr = "tag-1";
- byte tagType = (byte)10;
- Tag tag = new ArrayBackedTag(tagType, tagStr);
-
- ExtendedCellBuilder cellBuilder = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY);
- cellBuilder.setRow(Bytes.toBytes("row1"));
- cellBuilder.setFamily(Bytes.toBytes("f1"));
- cellBuilder.setQualifier(Bytes.toBytes("q1"));
- cellBuilder.setValue(Bytes.toBytes("value1"));
- cellBuilder.setType(Cell.Type.Delete);
- cellBuilder.setTags(Collections.singletonList(tag));
- Cell cell = cellBuilder.build();
-
- ClientProtos.Result protoResult =
- ProtobufUtil.toResult(Result.create(Collections.singletonList(cell)));
- assertNotNull(protoResult);
- assertEquals(1, protoResult.getCellCount());
-
- CellProtos.Cell protoCell = protoResult.getCell(0);
- ExtendedCellBuilder decodedBuilder =
- ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY);
- Cell decodedCell = ProtobufUtil.toCell(decodedBuilder, protoCell);
- List<Tag> decodedTags = PrivateCellUtil.getTags(decodedCell);
- assertEquals(1, decodedTags.size());
- Tag decodedTag = decodedTags.get(0);
- assertEquals(tagType, decodedTag.getType());
- assertEquals(tagStr, Tag.getValueAsString(decodedTag));
- }
}
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index edcd3c5..7c02be0 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.util.MapReduceExtendedCell;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -711,7 +710,6 @@ public class Import extends Configured implements Tool {
// If there's a rename mapping for this CF, create a new KeyValue
byte[] newCfName = cfRenameMap.get(CellUtil.cloneFamily(kv));
if (newCfName != null) {
- List<Tag> tags = PrivateCellUtil.getTags(kv);
kv = new KeyValue(kv.getRowArray(), // row buffer
kv.getRowOffset(), // row offset
kv.getRowLength(), // row length
@@ -725,8 +723,7 @@ public class Import extends Configured implements Tool {
KeyValue.Type.codeToType(kv.getTypeByte()), // KV Type
kv.getValueArray(), // value buffer
kv.getValueOffset(), // value offset
- kv.getValueLength(), // value length
- tags.size() == 0 ? null: tags);
+ kv.getValueLength()); // value length
}
}
return kv;
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index 018b9b8..97933a4 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -34,13 +34,10 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -49,12 +46,10 @@ import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
@@ -63,19 +58,11 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.Import.KeyValueImporter;
-import org.apache.hadoop.hbase.mapreduce.Import.CellImporter;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
@@ -130,9 +117,6 @@ public class TestImportExport {
private static final long now = System.currentTimeMillis();
private final TableName EXPORT_TABLE = TableName.valueOf("export_table");
private final TableName IMPORT_TABLE = TableName.valueOf("import_table");
- public static final byte TEST_TAG_TYPE = (byte) (Tag.CUSTOM_TAG_TYPE_RANGE + 1);
- public static final String TEST_ATTR = "source_op";
- public static final String TEST_TAG = "test_tag";
@BeforeClass
public static void beforeClass() throws Throwable {
@@ -820,147 +804,4 @@ public class TestImportExport {
return isVisited;
}
}
-
- /**
- * Add cell tags to delete mutations, run export and import tool and
- * verify that tags are present in import table also.
- * @throws Throwable throws Throwable.
- */
- @Test
- public void testTagsAddition() throws Throwable {
- final TableName exportTable = TableName.valueOf(name.getMethodName());
- TableDescriptor desc = TableDescriptorBuilder
- .newBuilder(exportTable)
- .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA)
- .setMaxVersions(5)
- .setKeepDeletedCells(KeepDeletedCells.TRUE)
- .build())
- .setCoprocessor(MetadataController.class.getName())
- .build();
- UTIL.getAdmin().createTable(desc);
-
- Table exportT = UTIL.getConnection().getTable(exportTable);
-
- //Add first version of QUAL
- Put p = new Put(ROW1);
- p.addColumn(FAMILYA, QUAL, now, QUAL);
- exportT.put(p);
-
- //Add Delete family marker
- Delete d = new Delete(ROW1, now+3);
- // Add test attribute to delete mutation.
- d.setAttribute(TEST_ATTR, Bytes.toBytes(TEST_TAG));
- exportT.delete(d);
-
- // Run export too with KeyValueCodecWithTags as Codec. This will ensure that export tool
- // will use KeyValueCodecWithTags.
- String[] args = new String[] {
- "-D" + ExportUtils.RAW_SCAN + "=true",
- // This will make sure that codec will encode and decode tags in rpc call.
- "-Dhbase.client.rpc.codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags",
- exportTable.getNameAsString(),
- FQ_OUTPUT_DIR,
- "1000", // max number of key versions per key to export
- };
- assertTrue(runExport(args));
- // Assert tag exists in exportTable
- assertTagExists(exportTable);
-
- // Create an import table with MetadataController.
- final TableName importTable = TableName.valueOf("importWithTestTagsAddition");
- TableDescriptor importTableDesc = TableDescriptorBuilder
- .newBuilder(importTable)
- .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA)
- .setMaxVersions(5)
- .setKeepDeletedCells(KeepDeletedCells.TRUE)
- .build())
- .setCoprocessor(MetadataController.class.getName())
- .build();
- UTIL.getAdmin().createTable(importTableDesc);
-
- // Run import tool.
- args = new String[] {
- // This will make sure that codec will encode and decode tags in rpc call.
- "-Dhbase.client.rpc.codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags",
- importTable.getNameAsString(),
- FQ_OUTPUT_DIR
- };
- assertTrue(runImport(args));
- // Make sure that tags exists in imported table.
- assertTagExists(importTable);
- }
-
- private void assertTagExists(TableName table) throws IOException {
- List<Cell> values = new ArrayList<>();
- for (HRegion region : UTIL.getHBaseCluster().getRegions(table)) {
- Scan scan = new Scan();
- // Make sure to set rawScan to true so that we will get Delete Markers.
- scan.setRaw(true);
- scan.readAllVersions();
- scan.withStartRow(ROW1);
- // Need to use RegionScanner instead of table#getScanner since the latter will
- // not return tags since it will go through rpc layer and remove tags intentionally.
- RegionScanner scanner = region.getScanner(scan);
- scanner.next(values);
- if (!values.isEmpty()) {
- break;
- }
- }
- boolean deleteFound = false;
- for (Cell cell: values) {
- if (PrivateCellUtil.isDelete(cell.getType().getCode())) {
- deleteFound = true;
- List<Tag> tags = PrivateCellUtil.getTags(cell);
- Assert.assertEquals(1, tags.size());
- for (Tag tag : tags) {
- Assert.assertEquals(TEST_TAG, Tag.getValueAsString(tag));
- }
- }
- }
- Assert.assertTrue(deleteFound);
- }
-
- /*
- This co-proc will add a cell tag to delete mutation.
- */
- public static class MetadataController implements RegionCoprocessor, RegionObserver {
- @Override
- public Optional<RegionObserver> getRegionObserver() {
- return Optional.of(this);
- }
-
- @Override
- public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
- MiniBatchOperationInProgress<Mutation> miniBatchOp)
- throws IOException {
- if (c.getEnvironment().getRegion().getRegionInfo().getTable().isSystemTable()) {
- return;
- }
- for (int i = 0; i < miniBatchOp.size(); i++) {
- Mutation m = miniBatchOp.getOperation(i);
- if (!(m instanceof Delete)) {
- continue;
- }
- byte[] sourceOpAttr = m.getAttribute(TEST_ATTR);
- if (sourceOpAttr == null) {
- continue;
- }
- Tag sourceOpTag = new ArrayBackedTag(TEST_TAG_TYPE, sourceOpAttr);
- List<Cell> updatedCells = new ArrayList<>();
- for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance(); ) {
- Cell cell = cellScanner.current();
- List<Tag> tags = PrivateCellUtil.getTags(cell);
- tags.add(sourceOpTag);
- Cell updatedCell = PrivateCellUtil.createCell(cell, tags);
- updatedCells.add(updatedCell);
- }
- m.getFamilyCellMap().clear();
- // Clear and add new Cells to the Mutation.
- for (Cell cell : updatedCells) {
- Delete d = (Delete) m;
- d.add(cell);
- }
- }
- }
- }
}