You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2018/10/24 17:17:19 UTC
hbase git commit: HBASE-21318 Make RefreshHFilesClient runnable
Repository: hbase
Updated Branches:
refs/heads/master 6830a1c1a -> d4cc5eef4
HBASE-21318 Make RefreshHFilesClient runnable
Other than when user enables hbase.coprocessor.region.classes with
RefreshHFilesEndPoint, user can also run this client as tool runner
class/CLI and calls refresh HFiles directly.
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d4cc5eef
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d4cc5eef
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d4cc5eef
Branch: refs/heads/master
Commit: d4cc5eef43b7e626c94f4a8aaabf63c17b9a0179
Parents: 6830a1c
Author: TAK LON WU <wu...@amazon.com>
Authored: Fri Aug 17 15:55:10 2018 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Oct 24 10:17:12 2018 -0700
----------------------------------------------------------------------
.../client/example/RefreshHFilesClient.java | 30 ++++++-
.../client/example/TestRefreshHFilesClient.java | 52 ++++++++++++
.../example/TestRefreshHFilesBase.java | 88 ++++++++++++++++++++
.../example/TestRefreshHFilesEndpoint.java | 64 +-------------
4 files changed, 171 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d4cc5eef/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
index ead0af0..06ad195 100644
--- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
+++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/RefreshHFilesClient.java
@@ -22,6 +22,8 @@ package org.apache.hadoop.hbase.client.example;
import java.io.Closeable;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
@@ -31,6 +33,8 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.protobuf.generated.RefreshHFilesProtos;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +44,7 @@ import org.slf4j.LoggerFactory;
* Region Server side via the RefreshHFilesService.
*/
@InterfaceAudience.Private
-public class RefreshHFilesClient implements Closeable {
+public class RefreshHFilesClient extends Configured implements Tool, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(RefreshHFilesClient.class);
private final Connection connection;
@@ -93,4 +97,28 @@ public class RefreshHFilesClient implements Closeable {
});
LOG.debug("Done refreshing HFiles");
}
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length != 1) {
+ String message = "When there are multiple HBase clusters are sharing a common root dir, "
+ + "especially for read replica cluster (see detail in HBASE-18477), please consider to "
+ + "use this tool manually sync the flushed HFiles from the source cluster.";
+ message += "\nUsage: " + this.getClass().getName() + " tableName";
+ System.out.println(message);
+ return -1;
+ }
+ final TableName tableName = TableName.valueOf(args[0]);
+ try {
+ refreshHFiles(tableName);
+ } catch (Throwable t) {
+ LOG.error("Refresh HFiles from table " + tableName.getNameAsString() + " failed: ", t);
+ return -1;
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ ToolRunner.run(new RefreshHFilesClient(HBaseConfiguration.create()), args);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d4cc5eef/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
new file mode 100644
index 0000000..b168db4
--- /dev/null
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/client/example/TestRefreshHFilesClient.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client.example;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.coprocessor.example.TestRefreshHFilesBase;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ClientTests.class, MediumTests.class })
+public class TestRefreshHFilesClient extends TestRefreshHFilesBase {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRefreshHFilesClient.class);
+
+ @BeforeClass
+ public static void setUp() {
+ setUp(HRegion.class.getName());
+ }
+
+ @Test
+ public void testRefreshHFilesClient() throws Exception {
+ addHFilesToRegions();
+ assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
+ RefreshHFilesClient tool = new RefreshHFilesClient(HTU.getConfiguration());
+ assertEquals(0, ToolRunner.run(tool, new String[] { TABLE_NAME.getNameAsString() }));
+ assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d4cc5eef/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
new file mode 100644
index 0000000..b948b62
--- /dev/null
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesBase.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor.example;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileTestUtil;
+import org.junit.After;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRefreshHFilesBase {
+ protected static final Logger LOG = LoggerFactory.getLogger(TestRefreshHFilesBase.class);
+ protected static final HBaseTestingUtility HTU = new HBaseTestingUtility();
+ protected static final int NUM_RS = 2;
+ protected static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP");
+ protected static final byte[] FAMILY = Bytes.toBytes("family");
+ protected static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
+ protected static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") };
+ protected static final int NUM_ROWS = 5;
+ protected static final String HFILE_NAME = "123abcdef";
+
+ protected static Configuration CONF = HTU.getConfiguration();
+ protected static MiniHBaseCluster cluster;
+ protected static Table table;
+
+ public static void setUp(String regionImpl) {
+ try {
+ CONF.set(HConstants.REGION_IMPL, regionImpl);
+ CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
+
+ CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName());
+ cluster = HTU.startMiniCluster(NUM_RS);
+
+ // Create table
+ table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);
+
+ // this will create 2 regions spread across slaves
+ HTU.loadNumericRows(table, FAMILY, 1, 20);
+ HTU.flush(TABLE_NAME);
+ } catch (Exception ex) {
+ LOG.error("Couldn't finish setup", ex);
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ HTU.shutdownMiniCluster();
+ }
+
+ protected void addHFilesToRegions() throws IOException {
+ MasterFileSystem mfs = HTU.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), TABLE_NAME);
+ for (Region region : cluster.getRegions(TABLE_NAME)) {
+ Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
+ Path familyDir = new Path(regionDir, Bytes.toString(FAMILY));
+ HFileTestUtil
+ .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir, HFILE_NAME), FAMILY,
+ QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d4cc5eef/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
index d942104..1a7c990 100644
--- a/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
+++ b/hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestRefreshHFilesEndpoint.java
@@ -27,92 +27,32 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.example.RefreshHFilesClient;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.apache.hadoop.hbase.wal.WAL;
-import org.junit.After;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@Category(MediumTests.class)
-public class TestRefreshHFilesEndpoint {
+public class TestRefreshHFilesEndpoint extends TestRefreshHFilesBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestRefreshHFilesEndpoint.class);
- private static final Logger LOG = LoggerFactory.getLogger(TestRefreshHFilesEndpoint.class);
- private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
- private static final int NUM_RS = 2;
- private static final TableName TABLE_NAME = TableName.valueOf("testRefreshRegionHFilesEP");
- private static final byte[] FAMILY = Bytes.toBytes("family");
- private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
- private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("30") };
- private static final int NUM_ROWS = 5;
- private static final String HFILE_NAME = "123abcdef";
-
- private static Configuration CONF = HTU.getConfiguration();
- private static MiniHBaseCluster cluster;
- private static Table table;
-
- public static void setUp(String regionImpl) {
- try {
- CONF.set(HConstants.REGION_IMPL, regionImpl);
- CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
-
- CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RefreshHFilesEndpoint.class.getName());
- cluster = HTU.startMiniCluster(NUM_RS);
-
- // Create table
- table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);
-
- // this will create 2 regions spread across slaves
- HTU.loadNumericRows(table, FAMILY, 1, 20);
- HTU.flush(TABLE_NAME);
- } catch (Exception ex) {
- LOG.error("Couldn't finish setup", ex);
- }
- }
-
- @After
- public void tearDown() throws Exception {
- HTU.shutdownMiniCluster();
- }
-
@Test
public void testRefreshRegionHFilesEndpoint() throws Exception {
setUp(HRegion.class.getName());
- MasterFileSystem mfs = HTU.getMiniHBaseCluster().getMaster().getMasterFileSystem();
- Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), TABLE_NAME);
- for (Region region : cluster.getRegions(TABLE_NAME)) {
- Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
- Path familyDir = new Path(regionDir, Bytes.toString(FAMILY));
- HFileTestUtil
- .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir, HFILE_NAME), FAMILY,
- QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS);
- }
+ addHFilesToRegions();
assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
callRefreshRegionHFilesEndPoint();
assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));