You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/01/04 17:56:46 UTC
[01/29] hbase git commit: HBASE-14030 HBase Backup/Restore Phase 1
(Vladimir Rodionov)
Repository: hbase
Updated Branches:
refs/heads/hbase-12439 5eefe1317 -> 00656688f
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
new file mode 100644
index 0000000..ad23f9b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestRestoreBoundaryTests extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestRestoreBoundaryTests.class);
+
+ /**
+ * Verify that a single empty table is restored to a new table
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingleEmpty() throws Exception {
+
+ LOG.info("test full restore on a single table empty table");
+ String backupId =
+ BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+ String[] tableset = new String[] { table1.getNameAsString() };
+ String[] tablemap = new String[] { table1_restore };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap,
+ false);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(TableName.valueOf(table1_restore)));
+ TEST_UTIL.deleteTable(TableName.valueOf(table1_restore));
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleEmpty() throws Exception {
+ LOG.info("create full backup image on multiple tables");
+ String tableset =
+ table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString();
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ assertTrue(checkSucceeded(backupId));
+ String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() };
+ String[] tablemap = new String[] { table2_restore, table3_restore };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, restore_tableset,
+ tablemap,
+ false);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(TableName.valueOf(table2_restore)));
+ assertTrue(hba.tableExists(TableName.valueOf(table3_restore)));
+ TEST_UTIL.deleteTable(TableName.valueOf(table2_restore));
+ TEST_UTIL.deleteTable(TableName.valueOf(table3_restore));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
index 7620bbb..cd2efad 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
@@ -49,7 +49,7 @@ public class SimpleRSProcedureManager extends RegionServerProcedureManager {
private ProcedureMember member;
@Override
- public void initialize(RegionServerServices rss) throws KeeperException {
+ public void initialize(RegionServerServices rss) throws IOException {
this.rss = rss;
ZooKeeperWatcher zkw = rss.getZooKeeper();
this.memberRpcs = new ZKProcedureMemberRpcs(zkw, getProcedureSignature());
[27/29] hbase git commit: Revert "HBASE-14888 ClusterSchema: Add
Namespace Operations"
Posted by sy...@apache.org.
Revert "HBASE-14888 ClusterSchema: Add Namespace Operations"
This reverts commit c5f3d17ae3a61cbf77cab89cddd8303e20e5e734.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a82f7fc9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a82f7fc9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a82f7fc9
Branch: refs/heads/hbase-12439
Commit: a82f7fc94a52e941b139367f1e67c03229213247
Parents: c5f3d17
Author: stack <st...@apache.org>
Authored: Sun Jan 3 07:15:18 2016 -0800
Committer: stack <st...@apache.org>
Committed: Sun Jan 3 07:15:18 2016 -0800
----------------------------------------------------------------------
.../hbase/client/ConnectionImplementation.java | 2 +-
.../java/org/apache/hadoop/hbase/Service.java | 50 ---
.../hbase/ServiceNotRunningException.java | 39 ---
.../apache/hadoop/hbase/ResourceChecker.java | 2 +-
.../hadoop/hbase/master/ClusterSchema.java | 153 ---------
.../hbase/master/ClusterSchemaException.java | 37 ---
.../hbase/master/ClusterSchemaService.java | 27 --
.../hbase/master/ClusterSchemaServiceImpl.java | 164 ----------
.../org/apache/hadoop/hbase/master/HMaster.java | 314 +++++++++++++------
.../hadoop/hbase/master/MasterRpcServices.java | 10 +-
.../hadoop/hbase/master/MasterServices.java | 78 ++++-
.../hadoop/hbase/master/ProcedureFuture.java | 132 --------
.../hbase/master/TableNamespaceManager.java | 68 ++--
.../procedure/CreateNamespaceProcedure.java | 2 +-
.../procedure/DeleteNamespaceProcedure.java | 2 +-
.../procedure/ModifyNamespaceProcedure.java | 3 +-
.../hbase/namespace/NamespaceStateManager.java | 6 +-
.../hbase/regionserver/HRegionServer.java | 3 +-
.../org/apache/hadoop/hbase/TestNamespace.java | 2 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 61 +++-
.../hbase/master/TestMasterNoCluster.java | 8 +
21 files changed, 401 insertions(+), 762 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 3498590..0ef2a17 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -398,7 +398,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
synchronized (this) {
if (batchPool == null) {
this.batchPool = getThreadPool(conf.getInt("hbase.hconnection.threads.max", 256),
- conf.getInt("hbase.hconnection.threads.core", 256), "-shared", null);
+ conf.getInt("hbase.hconnection.threads.core", 256), "-shared-", null);
this.cleanupPool = true;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java
deleted file mode 100644
index 97d93cc..0000000
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * Simple Service.
- */
-// This is a WIP. We have Services throughout hbase. Either have all implement what is here or
-// just remove this as an experiment that did not work out.
-// TODO: Move on to guava Service after we update our guava version; later guava has nicer
-// Service implmentation.
-// TODO: Move all Services on to this one Interface.
-@InterfaceAudience.Private
-public interface Service {
- /**
- * Initiates service startup (if necessary), returning once the service has finished starting.
- * @throws IOException Throws exception if already running and if we fail to start successfully.
- */
- void startAndWait() throws IOException;
-
- /**
- * @return True if this Service is running.
- */
- boolean isRunning();
-
- /**
- * Initiates service shutdown (if necessary), returning once the service has finished stopping.
- * @throws IOException Throws exception if not running of if we fail to stop successfully.
- */
- void stopAndWait() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java
deleted file mode 100644
index f6325ea..0000000
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.hadoop.hbase;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-@SuppressWarnings("serial")
-@InterfaceAudience.Private
-public class ServiceNotRunningException extends HBaseIOException {
- public ServiceNotRunningException() {
- }
-
- public ServiceNotRunningException(String message) {
- super(message);
- }
-
- public ServiceNotRunningException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ServiceNotRunningException(Throwable cause) {
- super(cause);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
index ee0380a..539aea3 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
@@ -41,7 +41,7 @@ public class ResourceChecker {
/**
* Constructor
- * @param tagLine The tagLine is added to the logs. Must not be null.
+ * @param tagLine - the tagLine is added to the logs. Must be be null.
*/
public ResourceChecker(final String tagLine) {
this.tagLine = tagLine;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java
deleted file mode 100644
index e632230..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.List;
-import java.util.concurrent.Future;
-
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.ProcedureInfo;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * View and edit the current cluster schema. Use this API making any modification to
- * namespaces, tables, etc.
- *
- * <h2>Implementation Notes</h2>
- * Nonces are for when operation is non-idempotent to ensure once-only semantic, even
- * across process failures.
- */
-// ClusterSchema is introduced to encapsulate schema modification. Currently the different aspects
-// are spread about the code base. This effort is about cleanup, shutting down access, and
-// coalescing common code. In particular, we'd contain filesystem modification. Other
-// benefits are to make all schema modification work the same way (one way to do an operation only
-// rather than the current approach where how an operation is done varies with context) and to make
-// it so clusterschema modification can stand apart from Master to faciliate standalone
-// testing. It is part of the filesystem refactor project that undoes the dependency on a
-// layout in HDFS that mimics our model of tables have regions have column families have files.
-// With this Interface in place, with all modifications going via this route where no filesystem
-// particulars are exposed, redoing our internals will take less effort.
-//
-// Currently ClusterSchema Interface will include namespace and table manipulation. Ideally a
-// form of this Interface will go all the ways down to the file manipulation level but currently
-// TBD.
-//
-// ClusterSchema is private to the Master; only the Master knows current cluster state and has
-// means of editing/altering it.
-//
-// TODO: Remove Server argument when MasterServices are passed.
-// TODO: We return Future<ProcedureInfo> in the below from most methods. It may change to return
-// a ProcedureFuture subsequently.
-@InterfaceAudience.Private
-public interface ClusterSchema {
- /**
- * Timeout for cluster operations in milliseconds.
- */
- public static final String HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY =
- "hbase.master.cluster.schema.operation.timeout";
- /**
- * Default operation timeout in milliseconds.
- */
- public static final int DEFAULT_HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT =
- 5 * 60 * 1000;
-
- /**
- * Utility method that will wait {@link #HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY}
- * timeout and if exceptions, THROWs the exception doing conversion so palatable outside Master:
- * i.e. {@link InterruptedException} becomes {@link InterruptedIOException} and so on.
- *
- * <<Utility>>
- *
- * @param future Future to wait on.
- * @return On completion, info on the procedure that ran.
- * @throws IOException
- */
- // TODO: Where to put this utility? It goes away?
- ProcedureInfo get(final Future<ProcedureInfo> future) throws IOException;
-
- /**
- * For internals use only. Do not use! Provisionally part of this Interface.
- * Prefer the high-level APIs available elsewhere in this API.
- * @return Instance of {@link TableNamespaceManager}
- */
- // TODO: Remove from here. Keep internal. This Interface is too high-level to host this accessor.
- TableNamespaceManager getTableNamespaceManager();
-
- /**
- * Create a new Namespace.
- * @param namespaceDescriptor descriptor for new Namespace
- * @param nonceGroup Identifier for the source of the request, a client or process.
- * @param nonce A unique identifier for this operation from the client or process identified by
- * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
- * @return Operation Future.
- * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion.
- * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
- * as well as {@link IOException}
- */
- Future<ProcedureInfo> createNamespace(NamespaceDescriptor namespaceDescriptor, long nonceGroup,
- long nonce)
- throws IOException;
-
- /**
- * Modify an existing Namespace.
- * @param nonceGroup Identifier for the source of the request, a client or process.
- * @param nonce A unique identifier for this operation from the client or process identified by
- * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
- * @return Operation Future.
- * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion.
- * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
- * as well as {@link IOException}
- */
- Future<ProcedureInfo> modifyNamespace(NamespaceDescriptor descriptor, long nonceGroup,
- long nonce)
- throws IOException;
-
- /**
- * Delete an existing Namespace.
- * Only empty Namespaces (no tables) can be removed.
- * @param nonceGroup Identifier for the source of the request, a client or process.
- * @param nonce A unique identifier for this operation from the client or process identified by
- * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
- * @return Operation Future.
- * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion.
- * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
- * as well as {@link IOException}
- */
- Future<ProcedureInfo> deleteNamespace(String name, long nonceGroup, long nonce)
- throws IOException;
-
- /**
- * Get a Namespace
- * @param name Name of the Namespace
- * @return Namespace descriptor for <code>name</code>
- * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
- * as well as {@link IOException}
- */
- // No Future here because presumption is that the request will go against cached metadata so
- // return immediately -- no need of running a Procedure.
- NamespaceDescriptor getNamespace(String name) throws IOException;
-
- /**
- * Get all Namespaces
- * @return All Namespace descriptors
- * @throws IOException
- */
- List<NamespaceDescriptor> getNamespaces() throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java
deleted file mode 100644
index 62892b6..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-@SuppressWarnings("serial")
-@InterfaceAudience.Private
-public class ClusterSchemaException extends HBaseIOException {
- public ClusterSchemaException(String message) {
- super(message);
- }
-
- public ClusterSchemaException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public ClusterSchemaException(Throwable cause) {
- super(cause);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java
deleted file mode 100644
index 43353ba..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import org.apache.hadoop.hbase.Service;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * Mixes in ClusterSchema and Service
- */
-@InterfaceAudience.Private
-public interface ClusterSchemaService extends ClusterSchema, Service {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java
deleted file mode 100644
index c7f4692..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.NamespaceNotFoundException;
-import org.apache.hadoop.hbase.ProcedureInfo;
-import org.apache.hadoop.hbase.ServiceNotRunningException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
-import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure;
-import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure;
-import org.apache.hadoop.hbase.procedure2.Procedure;
-import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-
-@InterfaceAudience.Private
-class ClusterSchemaServiceImpl implements ClusterSchemaService {
- private boolean running = false;
- private final TableNamespaceManager tableNamespaceManager;
- private final MasterServices masterServices;
- private final int clusterSchemaOperationTimeoutInMillis;
- private final static List<NamespaceDescriptor> EMPTY_NAMESPACE_LIST =
- Collections.unmodifiableList(new ArrayList<NamespaceDescriptor>(0));
-
- ClusterSchemaServiceImpl(final MasterServices masterServices) {
- this.masterServices = masterServices;
- this.tableNamespaceManager = new TableNamespaceManager(masterServices);
- this.clusterSchemaOperationTimeoutInMillis = this.masterServices.getConfiguration().
- getInt(HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY,
- DEFAULT_HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT);
- }
-
- // All below are synchronized so consistent view on whether running or not.
-
- @Override
- public synchronized boolean isRunning() {
- return this.running;
- }
-
- private synchronized void checkIsRunning() throws ServiceNotRunningException {
- if (!isRunning()) throw new ServiceNotRunningException();
- }
-
- @Override
- public synchronized void startAndWait() throws IOException {
- if (isRunning()) throw new IllegalStateException("Already running; cannot double-start.");
- // Set to running FIRST because tableNamespaceManager start uses this class to do namespace ops
- this.running = true;
- this.tableNamespaceManager.start();
- }
-
- @Override
- public synchronized void stopAndWait() throws IOException {
- checkIsRunning();
- // You can't stop tableNamespaceManager.
- this.running = false;
- }
-
- @Override
- public ProcedureInfo get(final Future<ProcedureInfo> future) throws IOException {
- try {
- ProcedureInfo pi =
- future.get(this.clusterSchemaOperationTimeoutInMillis, TimeUnit.MILLISECONDS);
- // If the procedure got an exception, throw it.
- if (pi.getException() != null) throw pi.getException();
- return pi;
- } catch (ExecutionException ee) {
- // No cleanup to do... just let the exception out.
- if (ee.getCause() instanceof IOException) throw (IOException)ee.getCause();
- else throw new ClusterSchemaException(ee.getCause());
- } catch (InterruptedException e) {
- IOException ioe = new InterruptedIOException();
- ioe.initCause(e);
- throw ioe;
- } catch (TimeoutException e) {
- throw new TimeoutIOException(future.toString());
- }
- }
-
- @Override
- public TableNamespaceManager getTableNamespaceManager() {
- return this.tableNamespaceManager;
- }
-
- private Future<ProcedureInfo> submitProcedure(final Procedure<?> procedure, long nonceGroup,
- long nonce)
- throws ServiceNotRunningException {
- checkIsRunning();
- ProcedureExecutor<MasterProcedureEnv> pe = this.masterServices.getMasterProcedureExecutor();
- long procId = pe.submitProcedure(procedure, nonceGroup, nonce);
- return new ProcedureFuture(pe, procId);
- }
-
- @Override
- public Future<ProcedureInfo> createNamespace(NamespaceDescriptor namespaceDescriptor,
- long nonceGroup, long nonce)
- throws IOException {
- return submitProcedure(new CreateNamespaceProcedure(
- this.masterServices.getMasterProcedureExecutor().getEnvironment(), namespaceDescriptor),
- nonceGroup, nonce);
- }
-
- @Override
- public Future<ProcedureInfo> modifyNamespace(NamespaceDescriptor namespaceDescriptor,
- long nonceGroup, long nonce)
- throws IOException {
- return submitProcedure(new ModifyNamespaceProcedure(
- this.masterServices.getMasterProcedureExecutor().getEnvironment(), namespaceDescriptor),
- nonceGroup, nonce);
- }
-
- @Override
- public Future<ProcedureInfo> deleteNamespace(String name, long nonceGroup, long nonce)
- throws IOException {
- return submitProcedure(new DeleteNamespaceProcedure(
- this.masterServices.getMasterProcedureExecutor().getEnvironment(), name),
- nonceGroup, nonce);
- }
-
- @Override
- public NamespaceDescriptor getNamespace(String name) throws IOException {
- NamespaceDescriptor nsd = getTableNamespaceManager().get(name);
- if (nsd == null) throw new NamespaceNotFoundException(name);
- return nsd;
- }
-
- @Override
- public List<NamespaceDescriptor> getNamespaces() throws IOException {
- checkIsRunning();
- Set<NamespaceDescriptor> set = getTableNamespaceManager().list();
- if (set == null || set.isEmpty()) return EMPTY_NAMESPACE_LIST;
- List<NamespaceDescriptor> list = new ArrayList<NamespaceDescriptor>(set.size());
- list.addAll(set);
- return Collections.unmodifiableList(list);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f29cad6..8e51f25 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -36,7 +36,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -64,6 +63,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.Server;
@@ -97,14 +97,17 @@ import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
+import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
+import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
+import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
@@ -182,7 +185,7 @@ import com.google.protobuf.Service;
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@SuppressWarnings("deprecation")
-public class HMaster extends HRegionServer implements MasterServices {
+public class HMaster extends HRegionServer implements MasterServices, Server {
private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
/**
@@ -253,7 +256,8 @@ public class HMaster extends HRegionServer implements MasterServices {
// Tracker for region normalizer state
private RegionNormalizerTracker regionNormalizerTracker;
- private ClusterSchemaService clusterSchemaService;
+ /** Namespace stuff */
+ private TableNamespaceManager tableNamespaceManager;
// Metrics for the HMaster
final MetricsMaster metricsMaster;
@@ -364,6 +368,9 @@ public class HMaster extends HRegionServer implements MasterServices {
* Remaining steps of initialization occur in
* #finishActiveMasterInitialization(MonitoredTask) after
* the master becomes the active one.
+ *
+ * @throws KeeperException
+ * @throws IOException
*/
public HMaster(final Configuration conf, CoordinatedStateManager csm)
throws IOException, KeeperException {
@@ -563,6 +570,10 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* Initialize all ZK based system trackers.
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws KeeperException
+ * @throws CoordinatedStateException
*/
void initializeZKBasedSystemTrackers() throws IOException,
InterruptedException, KeeperException, CoordinatedStateException {
@@ -619,6 +630,11 @@ public class HMaster extends HRegionServer implements MasterServices {
* <li>Ensure assignment of meta/namespace regions<li>
* <li>Handle either fresh cluster start or master failover</li>
* </ol>
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws KeeperException
+ * @throws CoordinatedStateException
*/
private void finishActiveMasterInitialization(MonitoredTask status)
throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
@@ -765,8 +781,8 @@ public class HMaster extends HRegionServer implements MasterServices {
this.catalogJanitorChore = new CatalogJanitor(this, this);
getChoreService().scheduleChore(catalogJanitorChore);
- status.setStatus("Starting cluster schema service");
- initClusterSchemaService();
+ status.setStatus("Starting namespace manager");
+ initNamespace();
if (this.cpHost != null) {
try {
@@ -832,6 +848,11 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* Create a {@link ServerManager} instance.
+ * @param master
+ * @param services
+ * @return An instance of {@link ServerManager}
+ * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
+ * @throws IOException
*/
ServerManager createServerManager(final Server master,
final MasterServices services)
@@ -867,6 +888,12 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* Check <code>hbase:meta</code> is assigned. If not, assign it.
+ * @param status MonitoredTask
+ * @param previouslyFailedMetaRSs
+ * @param replicaId
+ * @throws InterruptedException
+ * @throws IOException
+ * @throws KeeperException
*/
void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs, int replicaId)
throws InterruptedException, IOException, KeeperException {
@@ -938,10 +965,10 @@ public class HMaster extends HRegionServer implements MasterServices {
status.setStatus("META assigned.");
}
- void initClusterSchemaService() throws IOException, InterruptedException {
- this.clusterSchemaService = new ClusterSchemaServiceImpl(this);
- this.clusterSchemaService.startAndWait();
- if (!this.clusterSchemaService.isRunning()) throw new HBaseIOException("Failed start");
+ void initNamespace() throws IOException {
+ //create namespace manager
+ tableNamespaceManager = new TableNamespaceManager(this);
+ tableNamespaceManager.start();
}
void initQuotaManager() throws IOException {
@@ -987,6 +1014,7 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* This function returns a set of region server names under hbase:meta recovering region ZK node
* @return Set of meta server names which were recorded in ZK
+ * @throws KeeperException
*/
private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
Set<ServerName> result = new HashSet<ServerName>();
@@ -1022,6 +1050,11 @@ public class HMaster extends HRegionServer implements MasterServices {
return tableStateManager;
}
+ @Override
+ public TableNamespaceManager getTableNamespaceManager() {
+ return tableNamespaceManager;
+ }
+
/*
* Start up all services. If any of these threads gets an unhandled exception
* then they just die with a logged message. This should be fine because
@@ -1168,6 +1201,7 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* @return Get remote side's InetAddress
+ * @throws UnknownHostException
*/
InetAddress getRemoteInetAddress(final int port,
final long serverStartCode) throws UnknownHostException {
@@ -1304,6 +1338,7 @@ public class HMaster extends HRegionServer implements MasterServices {
* @return true if normalization step was performed successfully, false otherwise
* (specifically, if HMaster hasn't been initialized properly or normalization
* is globally disabled)
+ * @throws IOException
*/
public boolean normalizeRegions() throws IOException {
if (!this.initialized) {
@@ -1443,9 +1478,9 @@ public class HMaster extends HRegionServer implements MasterServices {
if (isStopped()) {
throw new MasterNotRunningException();
}
- checkInitialized();
+
String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
- this.clusterSchemaService.getNamespace(namespace);
+ ensureNamespaceExists(namespace);
HRegionInfo[] newRegions = ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
checkInitialized();
@@ -2270,9 +2305,18 @@ public class HMaster extends HRegionServer implements MasterServices {
void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
checkServiceStarted();
- if (!isInitialized()) throw new PleaseHoldException("Master is initializing");
+ if (!this.initialized) {
+ throw new PleaseHoldException("Master is initializing");
+ }
}
+ void checkNamespaceManagerReady() throws IOException {
+ checkInitialized();
+ if (tableNamespaceManager == null ||
+ !tableNamespaceManager.isTableAvailableAndInitialized(true)) {
+ throw new IOException("Table Namespace Manager not ready yet, try again later");
+ }
+ }
/**
* Report whether this master is currently the active master or not.
* If not active master, we are parked on ZK waiting to become active.
@@ -2367,6 +2411,7 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* Utility for constructing an instance of the passed HMaster class.
* @param masterClass
+ * @param conf
* @return HMaster instance.
*/
public static HMaster constructMaster(Class<? extends HMaster> masterClass,
@@ -2407,110 +2452,138 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public ClusterSchema getClusterSchema() {
- return this.clusterSchemaService;
+ public void createNamespace(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
+ checkNamespaceManagerReady();
+ if (cpHost != null) {
+ if (cpHost.preCreateNamespace(descriptor)) {
+ return;
+ }
+ }
+ createNamespaceSync(descriptor, nonceGroup, nonce);
+ if (cpHost != null) {
+ cpHost.postCreateNamespace(descriptor);
+ }
}
- /**
- * Create a new Namespace.
- * @param namespaceDescriptor descriptor for new Namespace
- * @param nonceGroup Identifier for the source of the request, a client or process.
- * @param nonce A unique identifier for this operation from the client or process identified by
- * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
- */
- void createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
- final long nonce)
- throws IOException {
- checkInitialized();
- TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
- if (this.cpHost != null && this.cpHost.preCreateNamespace(namespaceDescriptor)) return;
- LOG.info(getClientIdAuditPrefix() + " creating " + namespaceDescriptor);
+ @Override
+ public void createNamespaceSync(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
// Execute the operation synchronously - wait for the operation to complete before continuing.
- Future<ProcedureInfo> future =
- getClusterSchema().createNamespace(namespaceDescriptor, nonceGroup, nonce);
- getClusterSchema().get(future);
- if (this.cpHost != null) this.cpHost.postCreateNamespace(namespaceDescriptor);
+ long procId = this.procedureExecutor.submitProcedure(
+ new CreateNamespaceProcedure(procedureExecutor.getEnvironment(), descriptor),
+ nonceGroup,
+ nonce);
+ ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
}
- /**
- * Modify an existing Namespace.
- * @param nonceGroup Identifier for the source of the request, a client or process.
- * @param nonce A unique identifier for this operation from the client or process identified by
- * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).\
- */
- void modifyNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
- final long nonce)
- throws IOException {
- checkInitialized();
- TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
- if (this.cpHost != null && this.cpHost.preModifyNamespace(namespaceDescriptor)) return;
- LOG.info(getClientIdAuditPrefix() + " modify " + namespaceDescriptor);
+ @Override
+ public void modifyNamespace(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
+ checkNamespaceManagerReady();
+ if (cpHost != null) {
+ if (cpHost.preModifyNamespace(descriptor)) {
+ return;
+ }
+ }
+ LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
// Execute the operation synchronously - wait for the operation to complete before continuing.
- Future<ProcedureInfo> future =
- getClusterSchema().modifyNamespace(namespaceDescriptor, nonceGroup, nonce);
- getClusterSchema().get(future);
- if (this.cpHost != null) this.cpHost.postModifyNamespace(namespaceDescriptor);
+ long procId = this.procedureExecutor.submitProcedure(
+ new ModifyNamespaceProcedure(procedureExecutor.getEnvironment(), descriptor),
+ nonceGroup,
+ nonce);
+ ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
+ if (cpHost != null) {
+ cpHost.postModifyNamespace(descriptor);
+ }
}
- /**
- * Delete an existing Namespace. Only empty Namespaces (no tables) can be removed.
- * @param nonceGroup Identifier for the source of the request, a client or process.
- * @param nonce A unique identifier for this operation from the client or process identified by
- * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
- * @throws IOException
- */
- void deleteNamespace(final String name, final long nonceGroup, final long nonce)
- throws IOException {
- checkInitialized();
- if (this.cpHost != null && this.cpHost.preDeleteNamespace(name)) return;
+ @Override
+ public void deleteNamespace(
+ final String name,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ checkNamespaceManagerReady();
+ if (cpHost != null) {
+ if (cpHost.preDeleteNamespace(name)) {
+ return;
+ }
+ }
LOG.info(getClientIdAuditPrefix() + " delete " + name);
// Execute the operation synchronously - wait for the operation to complete before continuing.
- Future<ProcedureInfo> future = getClusterSchema().deleteNamespace(name, nonceGroup, nonce);
- getClusterSchema().get(future);
- if (this.cpHost != null) this.cpHost.postDeleteNamespace(name);
+ long procId = this.procedureExecutor.submitProcedure(
+ new DeleteNamespaceProcedure(procedureExecutor.getEnvironment(), name),
+ nonceGroup,
+ nonce);
+ ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
+ if (cpHost != null) {
+ cpHost.postDeleteNamespace(name);
+ }
}
/**
- * Get a Namespace
- * @param name Name of the Namespace
- * @return Namespace descriptor for <code>name</code>
+ * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException
+ *
+ * @param name the namespace to check
+ * @throws IOException if the namespace manager is not ready yet.
+ * @throws NamespaceNotFoundException if the namespace does not exists
*/
- NamespaceDescriptor getNamespace(String name) throws IOException {
- checkInitialized();
- if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name);
- NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name);
- if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd);
- return nsd;
+ private void ensureNamespaceExists(final String name)
+ throws IOException, NamespaceNotFoundException {
+ checkNamespaceManagerReady();
+ NamespaceDescriptor nsd = tableNamespaceManager.get(name);
+ if (nsd == null) {
+ throw new NamespaceNotFoundException(name);
+ }
}
- /**
- * Get all Namespaces
- * @return All Namespace descriptors
- */
- List<NamespaceDescriptor> getNamespaces() throws IOException {
- checkInitialized();
- final List<NamespaceDescriptor> nsds = new ArrayList<NamespaceDescriptor>();
- boolean bypass = false;
+ @Override
+ public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
+ checkNamespaceManagerReady();
+
if (cpHost != null) {
- bypass = cpHost.preListNamespaceDescriptors(nsds);
+ cpHost.preGetNamespaceDescriptor(name);
}
- if (!bypass) {
- nsds.addAll(this.clusterSchemaService.getNamespaces());
- if (this.cpHost != null) this.cpHost.postListNamespaceDescriptors(nsds);
+
+ NamespaceDescriptor nsd = tableNamespaceManager.get(name);
+ if (nsd == null) {
+ throw new NamespaceNotFoundException(name);
}
- return nsds;
- }
- @Override
- public List<TableName> listTableNamesByNamespace(String name) throws IOException {
- checkInitialized();
- return listTableNames(name, null, true);
+ if (cpHost != null) {
+ cpHost.postGetNamespaceDescriptor(nsd);
+ }
+
+ return nsd;
}
@Override
- public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
- checkInitialized();
- return listTableDescriptors(name, null, null, true);
+ public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
+ checkNamespaceManagerReady();
+
+ final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
+ boolean bypass = false;
+ if (cpHost != null) {
+ bypass = cpHost.preListNamespaceDescriptors(descriptors);
+ }
+
+ if (!bypass) {
+ descriptors.addAll(tableNamespaceManager.list());
+
+ if (cpHost != null) {
+ cpHost.postListNamespaceDescriptors(descriptors);
+ }
+ }
+ return descriptors;
}
@Override
@@ -2544,6 +2617,18 @@ public class HMaster extends HRegionServer implements MasterServices {
return procInfoList;
}
+ @Override
+ public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
+ ensureNamespaceExists(name);
+ return listTableDescriptors(name, null, null, true);
+ }
+
+ @Override
+ public List<TableName> listTableNamesByNamespace(String name) throws IOException {
+ ensureNamespaceExists(name);
+ return listTableNames(name, null, true);
+ }
+
/**
* Returns the list of table descriptors that match the specified request
*
@@ -2568,8 +2653,6 @@ public class HMaster extends HRegionServer implements MasterServices {
// request for all TableDescriptors
Collection<HTableDescriptor> htds;
if (namespace != null && namespace.length() > 0) {
- // Do a check on the namespace existence. Will fail if does not exist.
- this.clusterSchemaService.getNamespace(namespace);
htds = tableDescriptors.getByNamespace(namespace).values();
} else {
htds = tableDescriptors.getAll().values();
@@ -2613,17 +2696,46 @@ public class HMaster extends HRegionServer implements MasterServices {
*/
public List<TableName> listTableNames(final String namespace, final String regex,
final boolean includeSysTables) throws IOException {
- List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
- boolean bypass = cpHost != null? cpHost.preGetTableNames(htds, regex): false;
+ final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
+
+ boolean bypass = false;
+ if (cpHost != null) {
+ bypass = cpHost.preGetTableNames(descriptors, regex);
+ }
+
if (!bypass) {
- htds.addAll(listTableDescriptors(namespace, regex, null, includeSysTables));
- if (cpHost != null) cpHost.postGetTableNames(htds, regex);
+ // get all descriptors
+ Collection<HTableDescriptor> htds;
+ if (namespace != null && namespace.length() > 0) {
+ htds = tableDescriptors.getByNamespace(namespace).values();
+ } else {
+ htds = tableDescriptors.getAll().values();
+ }
+
+ for (HTableDescriptor htd: htds) {
+ if (includeSysTables || !htd.getTableName().isSystemTable()) {
+ descriptors.add(htd);
+ }
+ }
+
+ // Retains only those matched by regular expression.
+ if (regex != null) {
+ filterTablesByRegex(descriptors, Pattern.compile(regex));
+ }
+
+ if (cpHost != null) {
+ cpHost.postGetTableNames(descriptors, regex);
+ }
+ }
+
+ List<TableName> result = new ArrayList<TableName>(descriptors.size());
+ for (HTableDescriptor htd: descriptors) {
+ result.add(htd.getTableName());
}
- List<TableName> result = new ArrayList<TableName>(htds.size());
- for (HTableDescriptor htd: htds) result.add(htd.getTableName());
return result;
}
+
/**
* Removes the table descriptors that don't match the pattern.
* @param descriptors list of table descriptors to filter
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 2528814..b269c3d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.NamespaceExistException;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.ServerLoad;
@@ -833,7 +832,7 @@ public class MasterRpcServices extends RSRpcServices
try {
return GetNamespaceDescriptorResponse.newBuilder()
.setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(
- master.getNamespace(request.getNamespaceName())))
+ master.getNamespaceDescriptor(request.getNamespaceName())))
.build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -1121,7 +1120,7 @@ public class MasterRpcServices extends RSRpcServices
try {
ListNamespaceDescriptorsResponse.Builder response =
ListNamespaceDescriptorsResponse.newBuilder();
- for(NamespaceDescriptor ns: master.getNamespaces()) {
+ for(NamespaceDescriptor ns: master.listNamespaceDescriptors()) {
response.addNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(ns));
}
return response.build();
@@ -1306,9 +1305,10 @@ public class MasterRpcServices extends RSRpcServices
master.checkInitialized();
master.snapshotManager.checkSnapshotSupport();
- // Ensure namespace exists. Will throw exception if non-known NS.
+ // ensure namespace exists
TableName dstTable = TableName.valueOf(request.getSnapshot().getTable());
- master.getNamespace(dstTable.getNamespaceAsString());
+ master.getNamespaceDescriptor(dstTable.getNamespaceAsString());
+
SnapshotDescription reqSnapshot = request.getSnapshot();
master.snapshotManager.restoreSnapshot(reqSnapshot);
return RestoreSnapshotResponse.newBuilder().build();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index ec7db0c..af0e490 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -21,20 +21,21 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import com.google.protobuf.Service;
@@ -45,11 +46,6 @@ import com.google.protobuf.Service;
@InterfaceAudience.Private
public interface MasterServices extends Server {
/**
- * @return Master's instance of {@link ClusterSchema}
- */
- ClusterSchema getClusterSchema();
-
- /**
* @return Master's instance of the {@link AssignmentManager}
*/
AssignmentManager getAssignmentManager();
@@ -85,6 +81,11 @@ public interface MasterServices extends Server {
MasterCoprocessorHost getMasterCoprocessorHost();
/**
+ * @return Master's instance of {@link TableNamespaceManager}
+ */
+ TableNamespaceManager getTableNamespaceManager();
+
+ /**
* @return Master's instance of {@link MasterQuotaManager}
*/
MasterQuotaManager getMasterQuotaManager();
@@ -279,6 +280,54 @@ public interface MasterServices extends Server {
boolean isInitialized();
/**
+ * Create a new namespace
+ * @param descriptor descriptor which describes the new namespace
+ * @param nonceGroup
+ * @param nonce
+ * @throws IOException
+ */
+ public void createNamespace(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException;
+
+ /**
+ * Create a new namespace synchronously.
+ * @param descriptor descriptor which describes the new namespace
+ * @param nonceGroup
+ * @param nonce
+ * @throws IOException
+ */
+ public void createNamespaceSync(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException;
+
+ /**
+ * Modify an existing namespace
+ * @param descriptor descriptor which updates the existing namespace
+ * @param nonceGroup
+ * @param nonce
+ * @throws IOException
+ */
+ public void modifyNamespace(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException;
+
+ /**
+ * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
+ * @param name namespace name
+ * @param nonceGroup
+ * @param nonce
+ * @throws IOException
+ */
+ public void deleteNamespace(
+ final String name,
+ final long nonceGroup,
+ final long nonce) throws IOException;
+
+ /**
* Abort a procedure.
* @param procId ID of the procedure
* @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?
@@ -289,6 +338,21 @@ public interface MasterServices extends Server {
throws IOException;
/**
+ * Get a namespace descriptor by name
+ * @param name name of namespace descriptor
+ * @return A descriptor
+ * @throws IOException
+ */
+ public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException;
+
+ /**
+ * List available namespace descriptors
+ * @return A descriptor
+ * @throws IOException
+ */
+ public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException;
+
+ /**
* List procedures
* @return procedure list
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java
deleted file mode 100644
index 0a849fc..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.hadoop.hbase.ProcedureInfo;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.procedure2.Procedure;
-import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * Map Future Interface on to Procedure result processing.
- */
-// Has no extra methods as of now beyond Future<ProcedureInfo>. Use #toString if you want to log
-// procId of procedure.
-// TODO: This should be in Procedure? Have it in master package for now. Lets out ProcedureInfo.
-// Implementation informed by HBaseAdmin#ProcedureFuture.
-@InterfaceAudience.Private
-class ProcedureFuture implements Future<ProcedureInfo> {
- // Save exception so we can rethrow if called again. Same for result.
- private ExecutionException exception = null;
- private ProcedureInfo result = null;
- private boolean done = false;
- private boolean cancelled = false;
- private final Long procId;
- private final ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
-
- ProcedureFuture(final ProcedureExecutor<MasterProcedureEnv> procedureExecutor,
- final long procId) {
- this.procedureExecutor = procedureExecutor;
- this.procId = procId;
- }
-
- @Override
- public String toString() {
- return "procId=" + this.procId;
- }
-
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- if (!this.cancelled) {
- this.cancelled = this.procedureExecutor.abort(this.procId, mayInterruptIfRunning);
- }
- return this.cancelled;
- }
-
- @Override
- public boolean isCancelled() {
- return this.cancelled;
- }
-
- @Override
- public boolean isDone() {
- return done;
- }
-
- /**
- * This method is unsupported. We will throw an UnsupportedOperationException. Only the lazy
- * would call this method because they can avoid thinking through implication of a Procedure that
- * might never return so this is disallowed. Use {@link #get(long, TimeUnit)}.
- */
- @Override
- public ProcedureInfo get() throws InterruptedException, ExecutionException {
- // TODO: should we ever spin forever?
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcedureInfo get(long timeout, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- if (!this.done) {
- // TODO: add this sort of facility to EnvironmentEdgeManager
- long deadlineTs = EnvironmentEdgeManager.currentTime() + unit.toMillis(timeout);
- try {
- this.result = waitProcedureResult(procId, deadlineTs);
- } catch (IOException e) {
- this.exception = new ExecutionException(e);
- }
- this.done = true;
- }
- if (exception != null) {
- throw exception;
- }
- return result;
- }
-
- /**
- * @param procId
- * @param deadlineTs
- * @return A ProcedureInfo instance or null if procedure not found.
- * @throws IOException
- * @throws TimeoutException
- * @throws InterruptedException
- */
- private ProcedureInfo waitProcedureResult(long procId, long deadlineTs)
- throws IOException, TimeoutException, InterruptedException {
- while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
- Pair<ProcedureInfo, Procedure> pair = this.procedureExecutor.getResultOrProcedure(procId);
- if (pair.getFirst() != null) {
- this.procedureExecutor.removeResult(procId);
- return pair.getFirst();
- } else {
- if (pair.getSecond() == null) return null;
- }
- // TODO: Add a wait.
- }
- throw new TimeoutException("The procedure " + procId + " is still running");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
index 07822fd..bbeaf76 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
@@ -27,17 +27,17 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZKNamespaceManager;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.constraint.ConstraintException;
+import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
@@ -55,12 +56,10 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import com.google.common.collect.Sets;
/**
- * This is a helper class used internally to manage the namespace metadata that is stored in
- * TableName.NAMESPACE_TABLE_NAME. It also mirrors updates to the ZK store by forwarding updates to
- * {@link org.apache.hadoop.hbase.ZKNamespaceManager}.
- *
- * WARNING: Do not use. Go via the higher-level {@link ClusterSchema} API instead. This manager
- * is likely to go aways anyways.
+ * This is a helper class used to manage the namespace
+ * metadata that is stored in TableName.NAMESPACE_TABLE_NAME
+ * It also mirrors updates to the ZK store by forwarding updates to
+ * {@link org.apache.hadoop.hbase.ZKNamespaceManager}
*/
@InterfaceAudience.Private
public class TableNamespaceManager {
@@ -91,7 +90,7 @@ public class TableNamespaceManager {
private long exclusiveLockTimeoutMs;
private long sharedLockTimeoutMs;
- TableNamespaceManager(MasterServices masterServices) {
+ public TableNamespaceManager(MasterServices masterServices) {
this.masterServices = masterServices;
this.conf = masterServices.getConfiguration();
@@ -114,7 +113,7 @@ public class TableNamespaceManager {
// Wait for the namespace table to be initialized.
long startTime = EnvironmentEdgeManager.currentTime();
int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
- while (!isTableAvailableAndInitialized()) {
+ while (!isTableAvailableAndInitialized(false)) {
if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) {
// We can't do anything if ns is not online.
throw new IOException("Timedout " + timeout + "ms waiting for namespace table to "
@@ -270,29 +269,16 @@ public class TableNamespaceManager {
}
/**
- * Create Namespace in a blocking manner; don't return till success.
- * Note, by-passes notifying coprocessors and name checks. Use for system namespaces only.
- * @throws IOException
- * @throws InterruptedException
- */
- private void createNamespace(final NamespaceDescriptor namespaceDescriptor)
- throws IOException {
- ClusterSchema clusterSchema = this.masterServices.getClusterSchema();
- clusterSchema.get(clusterSchema.
- createNamespace(namespaceDescriptor, HConstants.NO_NONCE, HConstants.NO_NONCE));
- }
-
- /**
* This method checks if the namespace table is assigned and then
- * tries to create its Table reference. If it was already created before, it also makes
+ * tries to create its HTable. If it was already created before, it also makes
* sure that the connection isn't closed.
* @return true if the namespace table manager is ready to serve, false
* otherwise
* @throws IOException
*/
@SuppressWarnings("deprecation")
- public synchronized boolean isTableAvailableAndInitialized()
- throws IOException {
+ public synchronized boolean isTableAvailableAndInitialized(
+ final boolean createNamespaceAync) throws IOException {
// Did we already get a table? If so, still make sure it's available
if (isTableNamespaceManagerInitialized()) {
return true;
@@ -307,10 +293,34 @@ public class TableNamespaceManager {
zkNamespaceManager.start();
if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) {
- createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
+ if (createNamespaceAync) {
+ masterServices.getMasterProcedureExecutor().submitProcedure(
+ new CreateNamespaceProcedure(
+ masterServices.getMasterProcedureExecutor().getEnvironment(),
+ NamespaceDescriptor.DEFAULT_NAMESPACE));
+ initGoodSofar = false;
+ }
+ else {
+ masterServices.createNamespaceSync(
+ NamespaceDescriptor.DEFAULT_NAMESPACE,
+ HConstants.NO_NONCE,
+ HConstants.NO_NONCE);
+ }
}
if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) {
- createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
+ if (createNamespaceAync) {
+ masterServices.getMasterProcedureExecutor().submitProcedure(
+ new CreateNamespaceProcedure(
+ masterServices.getMasterProcedureExecutor().getEnvironment(),
+ NamespaceDescriptor.SYSTEM_NAMESPACE));
+ initGoodSofar = false;
+ }
+ else {
+ masterServices.createNamespaceSync(
+ NamespaceDescriptor.SYSTEM_NAMESPACE,
+ HConstants.NO_NONCE,
+ HConstants.NO_NONCE);
+ }
}
if (!initGoodSofar) {
@@ -400,4 +410,4 @@ public class TableNamespaceManager {
}
return maxRegions;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index f934737..657bbfb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -348,7 +348,7 @@ public class CreateNamespaceProcedure
}
private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) {
- return env.getMasterServices().getClusterSchema().getTableNamespaceManager();
+ return env.getMasterServices().getTableNamespaceManager();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index 2f99167..5a42614 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -383,7 +383,7 @@ public class DeleteNamespaceProcedure
}
private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) {
- return env.getMasterServices().getClusterSchema().getTableNamespaceManager();
+ return env.getMasterServices().getTableNamespaceManager();
}
/**
* The procedure could be restarted from a different machine. If the variable is null, we need to
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
index 0f8c172..30de252 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
@@ -266,9 +266,8 @@ public class ModifyNamespaceProcedure
}
private TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) {
- return env.getMasterServices().getClusterSchema().getTableNamespaceManager();
+ return env.getMasterServices().getTableNamespaceManager();
}
-
/**
* The procedure could be restarted from a different machine. If the variable is null, we need to
* retrieve it.
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
index 8035d32..f24f8c0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
@@ -65,7 +65,7 @@ class NamespaceStateManager {
/**
* Gets an instance of NamespaceTableAndRegionInfo associated with namespace.
- * @param name The name of the namespace
+ * @param The name of the namespace
* @return An instance of NamespaceTableAndRegionInfo.
*/
public NamespaceTableAndRegionInfo getState(String name) {
@@ -135,7 +135,7 @@ class NamespaceStateManager {
private NamespaceDescriptor getNamespaceDescriptor(String namespaceAsString) {
try {
- return this.master.getClusterSchema().getNamespace(namespaceAsString);
+ return this.master.getNamespaceDescriptor(namespaceAsString);
} catch (IOException e) {
LOG.error("Error while fetching namespace descriptor for namespace : " + namespaceAsString);
return null;
@@ -212,7 +212,7 @@ class NamespaceStateManager {
* Initialize namespace state cache by scanning meta table.
*/
private void initialize() throws IOException {
- List<NamespaceDescriptor> namespaces = this.master.getClusterSchema().getNamespaces();
+ List<NamespaceDescriptor> namespaces = this.master.listNamespaceDescriptors();
for (NamespaceDescriptor namespace : namespaces) {
addNamespace(namespace.getName());
List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1255fa4..211fed5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -196,7 +196,8 @@ import sun.misc.SignalHandler;
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@SuppressWarnings("deprecation")
-public class HRegionServer extends HasThread implements RegionServerServices, LastSequenceId {
+public class HRegionServer extends HasThread implements
+ RegionServerServices, LastSequenceId {
private static final Log LOG = LogFactory.getLog(HRegionServer.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
index f9e2a16..c24d8a3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
@@ -358,7 +358,7 @@ public class TestNamespace {
runWithExpectedException(new Callable<Void>() {
@Override
public Void call() throws Exception {
- admin.listTableDescriptorsByNamespace("non_existant_namespace");
+ admin.listTableDescriptorsByNamespace("non_existing_namespace");
return null;
}
}, NamespaceNotFoundException.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 785a006..8e35bbf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaMockingUtil;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.Server;
@@ -142,10 +143,11 @@ public class TestCatalogJanitor {
ServerName.valueOf("example.org,12345,6789"),
HRegionInfo.FIRST_META_REGIONINFO);
// Set hbase.rootdir into test dir.
- FileSystem.get(this.c);
+ FileSystem fs = FileSystem.get(this.c);
Path rootdir = FSUtils.getRootDir(this.c);
FSUtils.setRootDir(this.c, rootdir);
- Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
+ AdminProtos.AdminService.BlockingInterface hri =
+ Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
}
@Override
@@ -400,6 +402,48 @@ public class TestCatalogJanitor {
}
@Override
+ public void createNamespace(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
+ public void createNamespaceSync(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
+ public void modifyNamespace(
+ final NamespaceDescriptor descriptor,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
+ public void deleteNamespace(
+ final String name,
+ final long nonceGroup,
+ final long nonce) throws IOException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
+ public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
+ public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ @Override
public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
throws IOException {
return false; //To change body of implemented methods use File | Settings | File Templates.
@@ -492,29 +536,32 @@ public class TestCatalogJanitor {
}
@Override
+ public TableNamespaceManager getTableNamespaceManager() {
+ return null;
+ }
+
+ @Override
public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
boolean forcible) throws IOException {
}
@Override
public boolean isInitialized() {
+ // Auto-generated method stub
return false;
}
@Override
public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
+ // Auto-generated method stub
return 0;
}
@Override
public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
+ // Auto-generated method stub
return 0;
}
-
- @Override
- public ClusterSchema getClusterSchema() {
- return null;
- }
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/a82f7fc9/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index 20b492c..972834a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -218,6 +218,10 @@ public class TestMasterNoCluster {
return null;
}
}
+
+ @Override
+ void initNamespace() {
+ }
};
master.start();
@@ -290,6 +294,10 @@ public class TestMasterNoCluster {
return null;
}
}
+
+ @Override
+ void initNamespace() {
+ }
};
master.start();
[14/29] hbase git commit: Try this change to see how it effects
overall rating...
Posted by sy...@apache.org.
Try this change to see how it effects overall rating...
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3fca9a9f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3fca9a9f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3fca9a9f
Branch: refs/heads/hbase-12439
Commit: 3fca9a9f3fe56a870171492397f409cec500d3ac
Parents: 449fb81
Author: stack <st...@apache.org>
Authored: Thu Dec 24 17:23:33 2015 -0800
Committer: stack <st...@apache.org>
Committed: Mon Dec 28 13:23:23 2015 -0800
----------------------------------------------------------------------
dev-support/test-patch.sh | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3fca9a9f/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 509edeb..a7e75ea 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -857,13 +857,14 @@ runTests () {
{color:red}-1 core tests{color}. The patch failed these unit tests:
$failed_tests"
BAD=1
- JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+ #JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
else
JIRA_COMMENT="$JIRA_COMMENT
{color:green}+1 core tests{color}. The patch passed unit tests in $modules."
- JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
- BAD=$?
+ #JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+ #BAD=$?
+ BAD=0
fi
}
[25/29] hbase git commit: HBASE-15063 Bug in MultiByteBuf#toBytes.
(deepankar)
Posted by sy...@apache.org.
HBASE-15063 Bug in MultiByteBuf#toBytes. (deepankar)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f01a388a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f01a388a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f01a388a
Branch: refs/heads/hbase-12439
Commit: f01a388a35e03b432b1f3a1ecab04ab2c99e9444
Parents: 92abf8a
Author: anoopsjohn <an...@gmail.com>
Authored: Fri Jan 1 23:11:59 2016 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Fri Jan 1 23:11:59 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/nio/MultiByteBuff.java | 16 +--------
.../hadoop/hbase/nio/TestMultiByteBuff.java | 36 ++++++++++++++++++++
2 files changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f01a388a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
index 06652b8..ab2b5ea 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
@@ -1065,21 +1065,7 @@ public class MultiByteBuff extends ByteBuff {
@Override
public byte[] toBytes(int offset, int length) {
byte[] output = new byte[length];
- int itemIndex = getItemIndex(offset);
- ByteBuffer item = this.items[itemIndex];
- int toRead = item.limit() - offset;
- int destinationOffset = 0;
- while (length > 0) {
- toRead = Math.min(length, toRead);
- ByteBufferUtils.copyFromBufferToArray(output, item, offset, destinationOffset, toRead);
- length -= toRead;
- if (length == 0)
- break;
- destinationOffset += toRead;
- offset = 0;
- item = items[++itemIndex];
- toRead = item.remaining();
- }
+ this.get(offset, output, 0, length);
return output;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f01a388a/hbase-common/src/test/java/org/apache/hadoop/hbase/nio/TestMultiByteBuff.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/nio/TestMultiByteBuff.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/nio/TestMultiByteBuff.java
index 193fcff..800c8e1 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/nio/TestMultiByteBuff.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/nio/TestMultiByteBuff.java
@@ -342,4 +342,40 @@ public class TestMultiByteBuff {
assertEquals(2, dst[0]);
assertEquals(12, mbb1.position());
}
+
+ @Test
+ public void testToBytes() throws Exception {
+ byte[] b = new byte[4];
+ byte[] b1 = new byte[8];
+ for (int i = 0; i < b.length; i++) {
+ b[i] = (byte) i;
+ }
+ for (int i = 0; i < b1.length; i++) {
+ b1[i] = (byte) (b1.length + i);
+ }
+ ByteBuffer bb1 = ByteBuffer.wrap(b);
+ ByteBuffer bb2 = ByteBuffer.wrap(b1);
+ MultiByteBuff mbb1 = new MultiByteBuff(bb1, bb2);
+
+ // Test 1 Offset hitting exclusive second element
+ byte[] actual = mbb1.toBytes(6, 4);
+ assertTrue(Bytes.equals(actual, 0, actual.length,
+ b1, 2, 4));
+ // Test 2 offset hitting exclusive second element
+ // but continuing to the end of the second one
+ actual = mbb1.toBytes(5, 7);
+ assertTrue(Bytes.equals(actual, 0, actual.length,
+ b1, 1, 7));
+ // Test 3 with offset hitting in first element,
+ // continuing to next
+ actual = mbb1.toBytes(2, 7);
+ byte[] expected = new byte[7];
+ System.arraycopy(b, 2, expected, 0, 2);
+ System.arraycopy(b1, 0, expected, 2, 5);
+ assertTrue(Bytes.equals(actual, expected));
+ // Test 4 hitting only in first exclusively
+ actual = mbb1.toBytes(1, 3);
+ assertTrue(Bytes.equals(actual, 0, actual.length,
+ b, 1, 3));
+ }
}
[13/29] hbase git commit: Re-add zombie test and add return of result
of test running
Posted by sy...@apache.org.
Re-add zombie test and add return of result of test running
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a999c2a0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a999c2a0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a999c2a0
Branch: refs/heads/hbase-12439
Commit: a999c2a010296625509446cd4312f7b7d69954c1
Parents: 3fca9a9
Author: stack <st...@apache.org>
Authored: Mon Dec 28 13:22:49 2015 -0800
Committer: stack <st...@apache.org>
Committed: Mon Dec 28 13:23:23 2015 -0800
----------------------------------------------------------------------
dev-support/test-patch.sh | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a999c2a0/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index a7e75ea..d0c0346 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -856,15 +856,14 @@ runTests () {
{color:red}-1 core tests{color}. The patch failed these unit tests:
$failed_tests"
- BAD=1
- #JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+ JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+ return 1
else
JIRA_COMMENT="$JIRA_COMMENT
{color:green}+1 core tests{color}. The patch passed unit tests in $modules."
- #JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
- #BAD=$?
- BAD=0
+ JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+ return $?
fi
}
[19/29] hbase git commit: HBASE-15011 turn off the jdk8 javadoc
linter.
Posted by sy...@apache.org.
HBASE-15011 turn off the jdk8 javadoc linter.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/822fead7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/822fead7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/822fead7
Branch: refs/heads/hbase-12439
Commit: 822fead744a308df7ae45da440047207841d7abc
Parents: 9b8895b
Author: Sean Busbey <bu...@apache.org>
Authored: Sat Dec 26 16:47:53 2015 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 29 11:00:00 2015 -0600
----------------------------------------------------------------------
pom.xml | 27 ++++++++++++++++++++++++++-
1 file changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/822fead7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 08a1bf0..897b41c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -517,6 +517,11 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>${javadoc.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-remote-resources-plugin</artifactId>
<version>1.5</version>
</plugin>
@@ -1792,6 +1797,26 @@
-->
<profiles>
+ <profile>
+ <id>build-with-jdk8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <!-- TODO HBASE-15041 clean up our javadocs so jdk8 linter can be used -->
+ <additionalparam>-Xdoclint:none</additionalparam>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
<!-- profile activated by the Jenkins patch testing job -->
<profile>
<id>jenkins.patch</id>
@@ -2780,7 +2805,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.3</version>
+ <version>${javadoc.version}</version>
<reportSets>
<!-- Dev API -->
<reportSet>
[09/29] hbase git commit: HBASE-14030 Revert due to pending review
comments
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
deleted file mode 100644
index ae21b33..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
+++ /dev/null
@@ -1,496 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-/**
- * The main class which interprets the given arguments and trigger restore operation.
- */
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public final class RestoreClient {
-
- private static final Log LOG = LogFactory.getLog(RestoreClient.class);
-
- private static Options opt;
- private static Configuration conf;
- private static Set<BackupImage> lastRestoreImagesSet;
-
- // delimiter in tablename list in restore command
- private static final String DELIMITER_IN_COMMAND = ",";
-
- private static final String OPTION_OVERWRITE = "overwrite";
- private static final String OPTION_CHECK = "check";
- private static final String OPTION_AUTOMATIC = "automatic";
-
- private static final String USAGE =
- "Usage: hbase restore <backup_root_path> <backup_id> <tables> [tableMapping] \n"
- + " [-overwrite] [-check] [-automatic]\n"
- + " backup_root_path The parent location where the backup images are stored\n"
- + " backup_id The id identifying the backup image\n"
- + " table(s) Table(s) from the backup image to be restored.\n"
- + " Tables are separated by comma.\n"
- + " Options:\n"
- + " tableMapping A comma separated list of target tables.\n"
- + " If specified, each table in <tables> must have a mapping.\n"
- + " -overwrite With this option, restore overwrites to the existing table "
- + "if there's any in\n"
- + " restore target. The existing table must be online before restore.\n"
- + " -check With this option, restore sequence and dependencies are checked\n"
- + " and verified without executing the restore\n"
- + " -automatic With this option, all the dependencies are automatically restored\n"
- + " together with this backup image following the correct order.\n"
- + " The restore dependencies can be checked by using \"-check\" "
- + "option,\n"
- + " or using \"hbase backup describe\" command. Without this option, "
- + "only\n" + " this backup image is restored\n";
-
- private RestoreClient(){
- throw new AssertionError("Instantiating utility class...");
- }
-
- protected static void init() throws IOException {
- // define supported options
- opt = new Options();
- opt.addOption(OPTION_OVERWRITE, false,
- "Overwrite the data if any of the restore target tables exists");
- opt.addOption(OPTION_CHECK, false, "Check restore sequence and dependencies");
- opt.addOption(OPTION_AUTOMATIC, false, "Restore all dependencies");
- opt.addOption("debug", false, "Enable debug logging");
-
- conf = getConf();
-
- // disable irrelevant loggers to avoid it mess up command output
- disableUselessLoggers();
- }
-
- public static void main(String[] args) throws IOException {
- init();
- parseAndRun(args);
- }
-
- private static void parseAndRun(String[] args) {
- CommandLine cmd = null;
- try {
- cmd = new PosixParser().parse(opt, args);
- } catch (ParseException e) {
- LOG.error("Could not parse command", e);
- System.exit(-1);
- }
-
- // enable debug logging
- Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup");
- if (cmd.hasOption("debug")) {
- backupClientLogger.setLevel(Level.DEBUG);
- }
-
- // whether to overwrite to existing table if any, false by default
- boolean isOverwrite = cmd.hasOption(OPTION_OVERWRITE);
- if (isOverwrite) {
- LOG.debug("Found -overwrite option in restore command, "
- + "will overwrite to existing table if any in the restore target");
- }
-
- // whether to only check the dependencies, false by default
- boolean check = cmd.hasOption(OPTION_CHECK);
- if (check) {
- LOG.debug("Found -check option in restore command, "
- + "will check and verify the dependencies");
- }
-
- // whether to restore all dependencies, false by default
- boolean autoRestore = cmd.hasOption(OPTION_AUTOMATIC);
- if (autoRestore) {
- LOG.debug("Found -automatic option in restore command, "
- + "will automatically retore all the dependencies");
- }
-
- // parse main restore command options
- String[] remainArgs = cmd.getArgs();
- if (remainArgs.length < 3) {
- System.out.println("ERROR: missing arguments");
- System.out.println(USAGE);
- System.exit(-1);
- }
-
- String backupRootDir = remainArgs[0];
- String backupId = remainArgs[1];
- String tables = remainArgs[2];
-
- String tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null;
-
- String[] sTableArray = (tables != null) ? tables.split(DELIMITER_IN_COMMAND) : null;
- String[] tTableArray = (tableMapping != null) ? tableMapping.split(DELIMITER_IN_COMMAND) : null;
-
- if (tableMapping != null && tTableArray != null && (sTableArray.length != tTableArray.length)) {
- System.err.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping);
- System.out.println(USAGE);
- System.exit(-1);
- }
-
- try {
- HBackupFileSystem hBackupFS = new HBackupFileSystem(conf, new Path(backupRootDir), backupId);
- restore_stage1(hBackupFS, backupRootDir, backupId, check, autoRestore, sTableArray,
- tTableArray, isOverwrite);
- } catch (IOException e) {
- System.err.println("ERROR: " + e.getMessage());
- System.exit(-1);
- }
- }
-
- /**
- * Restore operation. Stage 1: validate backupManifest, and check target tables
- * @param hBackupFS to access the backup image
- * @param backupRootDir The root dir for backup image
- * @param backupId The backup id for image to be restored
- * @param check True if only do dependency check
- * @param autoRestore True if automatically restore following the dependency
- * @param sTableArray The array of tables to be restored
- * @param tTableArray The array of mapping tables to restore to
- * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the
- * request if target table exists
- * @return True if only do dependency check
- * @throws IOException if any failure during restore
- */
- public static boolean restore_stage1(HBackupFileSystem hBackupFS, String backupRootDir,
- String backupId, boolean check, boolean autoRestore, String[] sTableArray,
- String[] tTableArray, boolean isOverwrite) throws IOException {
-
- HashMap<String, BackupManifest> backupManifestMap = new HashMap<String, BackupManifest>();
- // check and load backup image manifest for the tables
- hBackupFS.checkImageManifestExist(backupManifestMap, sTableArray);
-
- try {
- // Check and validate the backup image and its dependencies
- if (check || autoRestore) {
- if (validate(backupManifestMap)) {
- LOG.info("Checking backup images: ok");
- } else {
- String errMsg = "Some dependencies are missing for restore";
- LOG.error(errMsg);
- throw new IOException(errMsg);
- }
- }
-
- // return true if only for check
- if (check) {
- return true;
- }
-
- if (tTableArray == null) {
- tTableArray = sTableArray;
- }
-
- // check the target tables
- checkTargetTables(tTableArray, isOverwrite);
-
- // start restore process
- Set<BackupImage> restoreImageSet =
- restore_stage2(hBackupFS, backupManifestMap, sTableArray, tTableArray, autoRestore);
-
- LOG.info("Restore for " + Arrays.asList(sTableArray) + " are successful!");
- lastRestoreImagesSet = restoreImageSet;
-
- } catch (IOException e) {
- LOG.error("ERROR: restore failed with error: " + e.getMessage());
- throw e;
- }
-
- // not only for check, return false
- return false;
- }
-
- /**
- * Get last restore image set. The value is globally set for the latest finished restore.
- * @return the last restore image set
- */
- public static Set<BackupImage> getLastRestoreImagesSet() {
- return lastRestoreImagesSet;
- }
-
- private static boolean validate(HashMap<String, BackupManifest> backupManifestMap)
- throws IOException {
- boolean isValid = true;
-
- for (Entry<String, BackupManifest> manifestEntry : backupManifestMap.entrySet()) {
-
- String table = manifestEntry.getKey();
- TreeSet<BackupImage> imageSet = new TreeSet<BackupImage>();
-
- ArrayList<BackupImage> depList = manifestEntry.getValue().getDependentListByTable(table);
- if (depList != null && !depList.isEmpty()) {
- imageSet.addAll(depList);
- }
-
- // todo merge
- LOG.debug("merge will be implemented in future jira");
- // BackupUtil.clearMergedImages(table, imageSet, conf);
-
- LOG.info("Dependent image(s) from old to new:");
- for (BackupImage image : imageSet) {
- String imageDir =
- HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table);
- if (!HBackupFileSystem.checkPathExist(imageDir, getConf())) {
- LOG.error("ERROR: backup image does not exist: " + imageDir);
- isValid = false;
- break;
- }
- // TODO More validation?
- LOG.info("Backup image: " + image.getBackupId() + " for '" + table + "' is available");
- }
- }
- return isValid;
- }
-
- /**
- * Validate target Tables
- * @param tTableArray: target tables
- * @param isOverwrite overwrite existing table
- * @throws IOException exception
- */
- private static void checkTargetTables(String[] tTableArray, boolean isOverwrite)
- throws IOException {
- ArrayList<String> existTableList = new ArrayList<String>();
- ArrayList<String> disabledTableList = new ArrayList<String>();
-
- // check if the tables already exist
- HBaseAdmin admin = null;
- Connection conn = null;
- try {
- conn = ConnectionFactory.createConnection(conf);
- admin = (HBaseAdmin) conn.getAdmin();
- for (String tableName : tTableArray) {
- if (admin.tableExists(TableName.valueOf(tableName))) {
- existTableList.add(tableName);
- if (admin.isTableDisabled(TableName.valueOf(tableName))) {
- disabledTableList.add(tableName);
- }
- } else {
- LOG.info("HBase table " + tableName
- + " does not exist. It will be create during backup process");
- }
- }
- } finally {
- if (admin != null) {
- admin.close();
- }
- if (conn != null) {
- conn.close();
- }
- }
-
- if (existTableList.size() > 0) {
- if (!isOverwrite) {
- LOG.error("Existing table found in the restore target, please add \"-overwrite\" "
- + "option in the command if you mean to restore to these existing tables");
- LOG.info("Existing table list in restore target: " + existTableList);
- throw new IOException("Existing table found in target while no \"-overwrite\" "
- + "option found");
- } else {
- if (disabledTableList.size() > 0) {
- LOG.error("Found offline table in the restore target, "
- + "please enable them before restore with \"-overwrite\" option");
- LOG.info("Offline table list in restore target: " + disabledTableList);
- throw new IOException(
- "Found offline table in the target when restore with \"-overwrite\" option");
- }
- }
- }
-
- }
-
- /**
- * Restore operation. Stage 2: resolved Backup Image dependency
- * @param hBackupFS to access the backup image
- * @param backupManifestMap : tableName, Manifest
- * @param sTableArray The array of tables to be restored
- * @param tTableArray The array of mapping tables to restore to
- * @param autoRestore : yes, restore all the backup images on the dependency list
- * @return set of BackupImages restored
- * @throws IOException exception
- */
- private static Set<BackupImage> restore_stage2(HBackupFileSystem hBackupFS,
- HashMap<String, BackupManifest> backupManifestMap, String[] sTableArray,
- String[] tTableArray, boolean autoRestore) throws IOException {
- TreeSet<BackupImage> restoreImageSet = new TreeSet<BackupImage>();
-
- for (int i = 0; i < sTableArray.length; i++) {
- restoreImageSet.clear();
- String table = sTableArray[i];
- BackupManifest manifest = backupManifestMap.get(table);
- if (autoRestore) {
- // Get the image list of this backup for restore in time order from old
- // to new.
- TreeSet<BackupImage> restoreList =
- new TreeSet<BackupImage>(manifest.getDependentListByTable(table));
- LOG.debug("need to clear merged Image. to be implemented in future jira");
-
- for (BackupImage image : restoreList) {
- restoreImage(image, table, tTableArray[i]);
- }
- restoreImageSet.addAll(restoreList);
- } else {
- BackupImage image = manifest.getBackupImage();
- List<BackupImage> depList = manifest.getDependentListByTable(table);
- // The dependency list always contains self.
- if (depList != null && depList.size() > 1) {
- LOG.warn("Backup image " + image.getBackupId() + " depends on other images.\n"
- + "this operation will only restore the delta contained within backupImage "
- + image.getBackupId());
- }
- restoreImage(image, table, tTableArray[i]);
- restoreImageSet.add(image);
- }
-
- if (autoRestore) {
- if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
- LOG.info("Restore includes the following image(s):");
- for (BackupImage image : restoreImageSet) {
- LOG.info(" Backup: "
- + image.getBackupId()
- + " "
- + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(),
- table));
- }
- }
- }
-
- }
- return restoreImageSet;
- }
-
- /**
- * Restore operation handle each backupImage
- * @param image: backupImage
- * @param sTable: table to be restored
- * @param tTable: table to be restored to
- * @throws IOException exception
- */
- private static void restoreImage(BackupImage image, String sTable, String tTable)
- throws IOException {
-
- Configuration conf = getConf();
-
- String rootDir = image.getRootDir();
- LOG.debug("Image root dir " + rootDir);
- String backupId = image.getBackupId();
-
- HBackupFileSystem hFS = new HBackupFileSystem(conf, new Path(rootDir), backupId);
- RestoreUtil restoreTool = new RestoreUtil(conf, hFS);
- BackupManifest manifest = hFS.getManifest(sTable);
-
- Path tableBackupPath = hFS.getTableBackupPath(sTable);
-
- // todo: convert feature will be provided in a future jira
- boolean converted = false;
-
- if (manifest.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL) || converted) {
- LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from "
- + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString());
- restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, converted);
- } else { // incremental Backup
- String logBackupDir =
- HBackupFileSystem.getLogBackupDir(image.getRootDir(), image.getBackupId());
- LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from incremental backup image "
- + logBackupDir);
- restoreTool.incrementalRestoreTable(logBackupDir, new String[] { sTable },
- new String[] { tTable });
- }
-
- LOG.info(sTable + " has been successfully restored to " + tTable);
- }
-
- /**
- * Set the configuration from a given one.
- * @param newConf A new given configuration
- */
- public synchronized static void setConf(Configuration newConf) {
- conf = newConf;
- }
-
- /**
- * Get and merge Hadoop and HBase configuration.
- * @throws IOException exception
- */
- protected static Configuration getConf() {
- if (conf == null) {
- synchronized (RestoreClient.class) {
- conf = new Configuration();
- HBaseConfiguration.merge(conf, HBaseConfiguration.create());
- }
- }
- return conf;
- }
-
- private static void disableUselessLoggers() {
- // disable zookeeper log to avoid it mess up command output
- Logger zkLogger = Logger.getLogger("org.apache.zookeeper");
- LOG.debug("Zookeeper log level before set: " + zkLogger.getLevel());
- zkLogger.setLevel(Level.OFF);
- LOG.debug("Zookeeper log level after set: " + zkLogger.getLevel());
-
- // disable hbase zookeeper tool log to avoid it mess up command output
- Logger hbaseZkLogger = Logger.getLogger("org.apache.hadoop.hbase.zookeeper");
- LOG.debug("HBase zookeeper log level before set: " + hbaseZkLogger.getLevel());
- hbaseZkLogger.setLevel(Level.OFF);
- LOG.debug("HBase Zookeeper log level after set: " + hbaseZkLogger.getLevel());
-
- // disable hbase client log to avoid it mess up command output
- Logger hbaseClientLogger = Logger.getLogger("org.apache.hadoop.hbase.client");
- LOG.debug("HBase client log level before set: " + hbaseClientLogger.getLevel());
- hbaseClientLogger.setLevel(Level.OFF);
- LOG.debug("HBase client log level after set: " + hbaseClientLogger.getLevel());
-
- // disable other related log to avoid mess up command output
- Logger otherLogger = Logger.getLogger("org.apache.hadoop.hbase.io.hfile");
- otherLogger.setLevel(Level.OFF);
- otherLogger = Logger.getLogger("org.apache.hadoop.hbase.util");
- otherLogger.setLevel(Level.OFF);
- otherLogger = Logger.getLogger("org.apache.hadoop.hbase.mapreduce");
- otherLogger.setLevel(Level.OFF);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java
deleted file mode 100644
index bdb7988..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java
+++ /dev/null
@@ -1,503 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.NavigableSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALFactory;
-import org.apache.hadoop.hbase.wal.WALKey;
-import org.apache.hadoop.hbase.wal.WALSplitter;
-
-/**
- * A collection for methods used by multiple classes to restore HBase tables.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class RestoreUtil {
-
- public static final Log LOG = LogFactory.getLog(RestoreUtil.class);
-
- protected Configuration conf = null;
-
- protected HBackupFileSystem hBackupFS = null;
-
- // store table name and snapshot dir mapping
- private final HashMap<String, Path> snapshotMap = new HashMap<String, Path>();
-
- public RestoreUtil(Configuration conf, HBackupFileSystem hBackupFS) throws IOException {
- this.conf = conf;
- this.hBackupFS = hBackupFS;
- }
-
- /**
- * During incremental backup operation. Call WalPlayer to replay WAL in backup image Currently
- * tableNames and newTablesNames only contain single table, will be expanded to multiple tables in
- * the future
- * @param logDir : incremental backup folders, which contains WAL
- * @param tableNames : source tableNames(table names were backuped)
- * @param newTableNames : target tableNames(table names to be restored to)
- * @throws IOException exception
- */
- public void incrementalRestoreTable(String logDir, String[] tableNames, String[] newTableNames)
- throws IOException {
-
- if (tableNames.length != newTableNames.length) {
- throw new IOException("Number of source tables adn taget Tables does not match!");
- }
-
- // for incremental backup image, expect the table already created either by user or previous
- // full backup. Here, check that all new tables exists
- HBaseAdmin admin = null;
- Connection conn = null;
- try {
- conn = ConnectionFactory.createConnection(conf);
- admin = (HBaseAdmin) conn.getAdmin();
- for (String tableName : newTableNames) {
- if (!admin.tableExists(TableName.valueOf(tableName))) {
- admin.close();
- throw new IOException("HBase table " + tableName
- + " does not exist. Create the table first, e.g. by restoring a full backup.");
- }
- }
- IncrementalRestoreService restoreService =
- BackupRestoreServiceFactory.getIncrementalRestoreService(conf);
-
- restoreService.run(logDir, tableNames, newTableNames);
- } finally {
- if (admin != null) {
- admin.close();
- }
- if(conn != null){
- conn.close();
- }
- }
- }
-
- public void fullRestoreTable(Path tableBackupPath, String tableName, String newTableName,
- boolean converted) throws IOException {
-
- restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted);
- }
-
- private void restoreTableAndCreate(String tableName, String newTableName, Path tableBackupPath,
- boolean converted) throws IOException {
- if (newTableName == null || newTableName.equals("")) {
- newTableName = tableName;
- }
-
- FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);
-
- // get table descriptor first
- HTableDescriptor tableDescriptor = null;
-
- Path tableSnapshotPath = hBackupFS.getTableSnapshotPath(tableName);
-
- if (fileSys.exists(tableSnapshotPath)) {
- // snapshot path exist means the backup path is in HDFS
- // check whether snapshot dir already recorded for target table
- if (snapshotMap.get(tableName) != null) {
- SnapshotDescription desc =
- SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
- SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
- tableDescriptor = manifest.getTableDescriptor();
- LOG.debug("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString()
- + " while tableName = " + tableName);
- // HBase 96.0 and 98.0
- // tableDescriptor =
- // FSTableDescriptors.getTableDescriptorFromFs(fileSys, snapshotMap.get(tableName));
- } else {
- tableDescriptor = hBackupFS.getTableDesc(tableName);
- LOG.debug("tableSnapshotPath=" + tableSnapshotPath.toString());
- snapshotMap.put(tableName, hBackupFS.getTableInfoPath(tableName));
- }
- if (tableDescriptor == null) {
- LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
- }
- } else if (converted) {
- // first check if this is a converted backup image
- LOG.error("convert will be supported in a future jira");
- }
-
- Path tableArchivePath = hBackupFS.getTableArchivePath(tableName);
- if (tableArchivePath == null) {
- if (tableDescriptor != null) {
- // find table descriptor but no archive dir means the table is empty, create table and exit
- LOG.debug("find table descriptor but no archive dir for table " + tableName
- + ", will only create table");
- tableDescriptor.setName(Bytes.toBytes(newTableName));
- checkAndCreateTable(tableBackupPath, tableName, newTableName, null, tableDescriptor);
- return;
- } else {
- throw new IllegalStateException("Cannot restore hbase table because directory '"
- + " tableArchivePath is null.");
- }
- }
-
- if (tableDescriptor == null) {
- tableDescriptor = new HTableDescriptor(newTableName);
- } else {
- tableDescriptor.setName(Bytes.toBytes(newTableName));
- }
-
- if (!converted) {
- // record all region dirs:
- // load all files in dir
- try {
- ArrayList<Path> regionPathList = hBackupFS.getRegionList(tableName);
-
- // should only try to create the table with all region informations, so we could pre-split
- // the regions in fine grain
- checkAndCreateTable(tableBackupPath, tableName, newTableName, regionPathList,
- tableDescriptor);
- if (tableArchivePath != null) {
- // start real restore through bulkload
- // if the backup target is on local cluster, special action needed
- Path tempTableArchivePath = hBackupFS.checkLocalAndBackup(tableArchivePath);
- if (tempTableArchivePath.equals(tableArchivePath)) {
- LOG.debug("TableArchivePath for bulkload using existPath: " + tableArchivePath);
- } else {
- regionPathList = hBackupFS.getRegionList(tempTableArchivePath); // point to the tempDir
- LOG.debug("TableArchivePath for bulkload using tempPath: " + tempTableArchivePath);
- }
-
- LoadIncrementalHFiles loader = createLoader(tempTableArchivePath, false);
- for (Path regionPath : regionPathList) {
- String regionName = regionPath.toString();
- LOG.debug("Restoring HFiles from directory " + regionName);
- String[] args = { regionName, newTableName };
- loader.run(args);
- }
- }
- // restore the recovered.edits if exists
- replayRecoveredEditsIfAny(tableBackupPath, tableName, tableDescriptor);
- } catch (Exception e) {
- throw new IllegalStateException("Cannot restore hbase table", e);
- }
- } else {
- LOG.debug("convert will be supported in a future jira");
- }
- }
-
- /**
- * Replay recovered edits from backup.
- */
- private void replayRecoveredEditsIfAny(Path tableBackupPath, String tableName,
- HTableDescriptor newTableHtd) throws IOException {
-
- LOG.debug("Trying to replay the recovered.edits if exist to the target table "
- + newTableHtd.getNameAsString() + " from the backup of table " + tableName + ".");
-
- FileSystem fs = tableBackupPath.getFileSystem(this.conf);
- ArrayList<Path> regionDirs = hBackupFS.getRegionList(tableName);
-
- if (regionDirs == null || regionDirs.size() == 0) {
- LOG.warn("No recovered.edits to be replayed for empty backup of table " + tableName + ".");
- return;
- }
-
- Connection conn = null;
- try {
-
- conn = ConnectionFactory.createConnection(conf);
-
- for (Path regionDir : regionDirs) {
- // OLD: NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regionDir);
- NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regionDir);
-
- if (files == null || files.isEmpty()) {
- LOG.warn("No recovered.edits found for the region " + regionDir.getName() + ".");
- return;
- }
-
- for (Path edits : files) {
- if (edits == null || !fs.exists(edits)) {
- LOG.warn("Null or non-existent edits file: " + edits);
- continue;
- }
-
- HTable table = null;
- try {
- table = (HTable) conn.getTable(newTableHtd.getTableName());
- replayRecoveredEdits(table, fs, edits);
- table.flushCommits();
- table.close();
- } catch (IOException e) {
- boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
- if (skipErrors) {
- Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
- LOG.error(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS
- + "=true so continuing. Renamed " + edits + " as " + p, e);
- } else {
- throw e;
- }
- } finally {
- if (table != null) {
- table.close();
- }
- }
- } // for each edit file under a region
- } // for each region
-
- } finally {
- if (conn != null) {
- conn.close();
- }
- }
- }
-
- /**
- * Restore process for an edit entry.
- * @param htable The target table of restore
- * @param key HLog key
- * @param val KVs
- * @throws IOException exception
- */
- private void restoreEdit(HTable htable, WALKey key, WALEdit val) throws IOException {
- Put put = null;
- Delete del = null;
- Cell lastKV = null;
- for (Cell kv : val.getCells()) {
- // filtering HLog meta entries, see HLog.completeCacheFlushLogEdit
- if (WALEdit.isMetaEditFamily(CellUtil.cloneFamily(kv))) {
- continue;
- }
-
- // A WALEdit may contain multiple operations (HBASE-3584) and/or
- // multiple rows (HBASE-5229).
- // Aggregate as much as possible into a single Put/Delete
- // operation before apply the action to the table.
- if (lastKV == null || lastKV.getTypeByte() != kv.getTypeByte()
- || !CellUtil.matchingRow(lastKV, kv)) {
- // row or type changed, write out aggregate KVs.
- if (put != null) {
- applyAction(htable, put);
- }
- if (del != null) {
- applyAction(htable, del);
- }
-
- if (CellUtil.isDelete(kv)) {
- del = new Delete(CellUtil.cloneRow(kv));
- } else {
- put = new Put(CellUtil.cloneRow(kv));
- }
- }
- if (CellUtil.isDelete(kv)) {
- del.addDeleteMarker(kv);
- } else {
- put.add(kv);
- }
- lastKV = kv;
- }
- // write residual KVs
- if (put != null) {
- applyAction(htable, put);
- }
- if (del != null) {
- applyAction(htable, del);
- }
- }
-
- /**
- * Apply an action (Put/Delete) to table.
- * @param table table
- * @param action action
- * @throws IOException exception
- */
- private void applyAction(HTable table, Mutation action) throws IOException {
- // The actions are not immutable, so we defensively copy them
- if (action instanceof Put) {
- Put put = new Put((Put) action);
- // put.setWriteToWAL(false);
- // why do not we do WAL?
- put.setDurability(Durability.SKIP_WAL);
- table.put(put);
- } else if (action instanceof Delete) {
- Delete delete = new Delete((Delete) action);
- table.delete(delete);
- } else {
- throw new IllegalArgumentException("action must be either Delete or Put");
- }
- }
-
- /**
- * Replay the given edits.
- * @param htable The target table of restore
- * @param fs File system
- * @param edits Recovered.edits to be replayed
- * @throws IOException exception
- */
- private void replayRecoveredEdits(HTable htable, FileSystem fs, Path edits) throws IOException {
- LOG.debug("Replaying edits from " + edits + "; path=" + edits);
-
- WAL.Reader reader = null;
- try {
- reader = WALFactory.createReader(fs, edits, this.conf);
- long editsCount = 0;
- WAL.Entry entry;
-
- try {
- while ((entry = reader.next()) != null) {
- restoreEdit(htable, entry.getKey(), entry.getEdit());
- editsCount++;
- }
- LOG.debug(editsCount + " edits from " + edits + " have been replayed.");
-
- } catch (EOFException eof) {
- Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
- String msg =
- "Encountered EOF. Most likely due to Master failure during "
- + "log spliting, so we have this data in another edit. "
- + "Continuing, but renaming " + edits + " as " + p;
- LOG.warn(msg, eof);
- } catch (IOException ioe) {
- // If the IOE resulted from bad file format,
- // then this problem is idempotent and retrying won't help
- if (ioe.getCause() instanceof ParseException) {
- Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
- String msg =
- "File corruption encountered! " + "Continuing, but renaming " + edits + " as " + p;
- LOG.warn(msg, ioe);
- } else {
- // other IO errors may be transient (bad network connection,
- // checksum exception on one datanode, etc). throw & retry
- throw ioe;
- }
- }
- } finally {
- if (reader != null) {
- reader.close();
- }
- }
- }
-
- /**
- * Create a {@link LoadIncrementalHFiles} instance to be used to restore the HFiles of a full
- * backup.
- * @return the {@link LoadIncrementalHFiles} instance
- * @throws IOException exception
- */
- private LoadIncrementalHFiles createLoader(Path tableArchivePath, boolean multipleTables)
- throws IOException {
- // set configuration for restore:
- // LoadIncrementalHFile needs more time
- // <name>hbase.rpc.timeout</name> <value>600000</value>
- // calculates
- Integer milliSecInMin = 60000;
- Integer previousMillis = this.conf.getInt("hbase.rpc.timeout", 0);
- Integer numberOfFilesInDir =
- multipleTables ? hBackupFS.getMaxNumberOfFilesInSubDir(tableArchivePath) : hBackupFS
- .getNumberOfFilesInDir(tableArchivePath);
- Integer calculatedMillis = numberOfFilesInDir * milliSecInMin; // 1 minute per file
- Integer resultMillis = Math.max(calculatedMillis, previousMillis);
- if (resultMillis > previousMillis) {
- LOG.info("Setting configuration for restore with LoadIncrementalHFile: "
- + "hbase.rpc.timeout to " + calculatedMillis / milliSecInMin
- + " minutes, to handle the number of files in backup " + tableArchivePath);
- this.conf.setInt("hbase.rpc.timeout", resultMillis);
- }
-
- LoadIncrementalHFiles loader = null;
- try {
- loader = new LoadIncrementalHFiles(this.conf);
- } catch (Exception e1) {
- throw new IOException(e1);
- }
- return loader;
- }
-
- /**
- * Prepare the table for bulkload, most codes copied from
- * {@link LoadIncrementalHFiles#createTable(String, String)}
- * @param tableBackupPath path
- * @param tableName table name
- * @param targetTableName target table name
- * @param regionDirList region directory list
- * @param htd table descriptor
- * @throws IOException exception
- */
- private void checkAndCreateTable(Path tableBackupPath, String tableName, String targetTableName,
- ArrayList<Path> regionDirList, HTableDescriptor htd) throws IOException {
- HBaseAdmin hbadmin = null;
- Connection conn = null;
- try {
- conn = ConnectionFactory.createConnection(conf);
- hbadmin = (HBaseAdmin) conn.getAdmin();
- if (hbadmin.tableExists(TableName.valueOf(targetTableName))) {
- LOG.info("Using exising target table '" + targetTableName + "'");
- } else {
- LOG.info("Creating target table '" + targetTableName + "'");
-
- // if no region dir given, create the table and return
- if (regionDirList == null || regionDirList.size() == 0) {
-
- hbadmin.createTable(htd);
- return;
- }
-
- byte[][] keys = hBackupFS.generateBoundaryKeys(regionDirList);
-
- // create table using table decriptor and region boundaries
- hbadmin.createTable(htd, keys);
- }
- } catch (Exception e) {
- throw new IOException(e);
- } finally {
- if (hbadmin != null) {
- hbadmin.close();
- }
- if(conn != null){
- conn.close();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java
deleted file mode 100644
index a3b5db5..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup.mapreduce;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.math.BigDecimal;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.backup.BackupCopyService;
-import org.apache.hadoop.hbase.backup.BackupHandler;
-import org.apache.hadoop.hbase.backup.BackupUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.tools.DistCp;
-import org.apache.hadoop.tools.DistCpConstants;
-import org.apache.hadoop.tools.DistCpOptions;
-/**
- * Copier for backup operation. Basically, there are 2 types of copy. One is copying from snapshot,
- * which bases on extending ExportSnapshot's function with copy progress reporting to ZooKeeper
- * implementation. The other is copying for incremental log files, which bases on extending
- * DistCp's function with copy progress reporting to ZooKeeper implementation.
- *
- * For now this is only a wrapper. The other features such as progress and increment backup will be
- * implemented in future jira
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class MapReduceBackupCopyService implements BackupCopyService {
- private static final Log LOG = LogFactory.getLog(MapReduceBackupCopyService.class);
-
- private Configuration conf;
- // private static final long BYTES_PER_MAP = 2 * 256 * 1024 * 1024;
-
- // Accumulated progress within the whole backup process for the copy operation
- private float progressDone = 0.1f;
- private long bytesCopied = 0;
- private static float INIT_PROGRESS = 0.1f;
-
- // The percentage of the current copy task within the whole task if multiple time copies are
- // needed. The default value is 100%, which means only 1 copy task for the whole.
- private float subTaskPercntgInWholeTask = 1f;
-
- public MapReduceBackupCopyService() {
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * Get the current copy task percentage within the whole task if multiple copies are needed.
- * @return the current copy task percentage
- */
- public float getSubTaskPercntgInWholeTask() {
- return subTaskPercntgInWholeTask;
- }
-
- /**
- * Set the current copy task percentage within the whole task if multiple copies are needed. Must
- * be called before calling
- * {@link #copy(BackupHandler, Configuration, Type, String[])}
- * @param subTaskPercntgInWholeTask The percentage of the copy subtask
- */
- public void setSubTaskPercntgInWholeTask(float subTaskPercntgInWholeTask) {
- this.subTaskPercntgInWholeTask = subTaskPercntgInWholeTask;
- }
-
- class SnapshotCopy extends ExportSnapshot {
- private BackupHandler backupHandler;
- private String table;
-
- public SnapshotCopy(BackupHandler backupHandler, String table) {
- super();
- this.backupHandler = backupHandler;
- this.table = table;
- }
-
- public BackupHandler getBackupHandler() {
- return this.backupHandler;
- }
-
- public String getTable() {
- return this.table;
- }
- }
-
- // Extends DistCp for progress updating to hbase:backup
- // during backup. Using DistCpV2 (MAPREDUCE-2765).
- // Simply extend it and override execute() method to get the
- // Job reference for progress updating.
- // Only the argument "src1, [src2, [...]] dst" is supported,
- // no more DistCp options.
- class BackupDistCp extends DistCp {
-
- private BackupHandler backupHandler;
-
- public BackupDistCp(Configuration conf, DistCpOptions options, BackupHandler backupHandler)
- throws Exception {
- super(conf, options);
- this.backupHandler = backupHandler;
- }
-
- @Override
- public Job execute() throws Exception {
-
- // reflection preparation for private methods and fields
- Class<?> classDistCp = org.apache.hadoop.tools.DistCp.class;
- Method methodCreateMetaFolderPath = classDistCp.getDeclaredMethod("createMetaFolderPath");
- Method methodCreateJob = classDistCp.getDeclaredMethod("createJob");
- Method methodCreateInputFileListing =
- classDistCp.getDeclaredMethod("createInputFileListing", Job.class);
- Method methodCleanup = classDistCp.getDeclaredMethod("cleanup");
-
- Field fieldInputOptions = classDistCp.getDeclaredField("inputOptions");
- Field fieldMetaFolder = classDistCp.getDeclaredField("metaFolder");
- Field fieldJobFS = classDistCp.getDeclaredField("jobFS");
- Field fieldSubmitted = classDistCp.getDeclaredField("submitted");
-
- methodCreateMetaFolderPath.setAccessible(true);
- methodCreateJob.setAccessible(true);
- methodCreateInputFileListing.setAccessible(true);
- methodCleanup.setAccessible(true);
-
- fieldInputOptions.setAccessible(true);
- fieldMetaFolder.setAccessible(true);
- fieldJobFS.setAccessible(true);
- fieldSubmitted.setAccessible(true);
-
- // execute() logic starts here
- assert fieldInputOptions.get(this) != null;
- assert getConf() != null;
-
- Job job = null;
- try {
- synchronized (this) {
- // Don't cleanup while we are setting up.
- fieldMetaFolder.set(this, methodCreateMetaFolderPath.invoke(this));
- fieldJobFS.set(this, ((Path) fieldMetaFolder.get(this)).getFileSystem(getConf()));
-
- job = (Job) methodCreateJob.invoke(this);
- }
- methodCreateInputFileListing.invoke(this, job);
-
- // Get the total length of the source files
- List<Path> srcs = ((DistCpOptions) fieldInputOptions.get(this)).getSourcePaths();
- long totalSrcLgth = 0;
- for (Path aSrc : srcs) {
- totalSrcLgth += BackupUtil.getFilesLength(aSrc.getFileSystem(getConf()), aSrc);
- }
-
- // submit the copy job
- job.submit();
- fieldSubmitted.set(this, true);
-
- // after submit the MR job, set its handler in backup handler for cancel process
- // this.backupHandler.copyJob = job;
-
- // Update the copy progress to ZK every 0.5s if progress value changed
- int progressReportFreq =
- this.getConf().getInt("hbase.backup.progressreport.frequency", 500);
- float lastProgress = progressDone;
- while (!job.isComplete()) {
- float newProgress =
- progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS);
-
- if (newProgress > lastProgress) {
-
- BigDecimal progressData =
- new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP);
- String newProgressStr = progressData + "%";
- LOG.info("Progress: " + newProgressStr);
- this.backupHandler.updateProgress(newProgressStr, bytesCopied);
- LOG.debug("Backup progress data updated to hbase:backup: \"Progress: " + newProgressStr
- + ".\"");
- lastProgress = newProgress;
- }
- Thread.sleep(progressReportFreq);
- }
-
- // update the progress data after copy job complete
- float newProgress =
- progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS);
- BigDecimal progressData =
- new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP);
-
- String newProgressStr = progressData + "%";
- LOG.info("Progress: " + newProgressStr);
-
- // accumulate the overall backup progress
- progressDone = newProgress;
- bytesCopied += totalSrcLgth;
-
- this.backupHandler.updateProgress(newProgressStr, bytesCopied);
- LOG.debug("Backup progress data updated to hbase:backup: \"Progress: " + newProgressStr
- + " - " + bytesCopied + " bytes copied.\"");
-
- } finally {
- if (!fieldSubmitted.getBoolean(this)) {
- methodCleanup.invoke(this);
- }
- }
-
- String jobID = job.getJobID().toString();
- job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, jobID);
-
- LOG.debug("DistCp job-id: " + jobID);
- return job;
- }
-
- }
-
- /**
- * Do backup copy based on different types.
- * @param handler The backup handler reference
- * @param conf The hadoop configuration
- * @param copyType The backup copy type
- * @param options Options for customized ExportSnapshot or DistCp
- * @throws Exception exception
- */
- public int copy(BackupHandler handler, Configuration conf, BackupCopyService.Type copyType,
- String[] options) throws IOException {
-
- int res = 0;
-
- try {
- if (copyType == Type.FULL) {
- SnapshotCopy snapshotCp =
- new SnapshotCopy(handler, handler.getBackupContext().getTableBySnapshot(options[1]));
- LOG.debug("Doing SNAPSHOT_COPY");
- // Make a new instance of conf to be used by the snapshot copy class.
- snapshotCp.setConf(new Configuration(conf));
- res = snapshotCp.run(options);
- } else if (copyType == Type.INCREMENTAL) {
- LOG.debug("Doing COPY_TYPE_DISTCP");
- setSubTaskPercntgInWholeTask(1f);
-
- BackupDistCp distcp = new BackupDistCp(new Configuration(conf), null, handler);
- // Handle a special case where the source file is a single file.
- // In this case, distcp will not create the target dir. It just take the
- // target as a file name and copy source file to the target (as a file name).
- // We need to create the target dir before run distcp.
- LOG.debug("DistCp options: " + Arrays.toString(options));
- if (options.length == 2) {
- Path dest = new Path(options[1]);
- FileSystem destfs = dest.getFileSystem(conf);
- if (!destfs.exists(dest)) {
- destfs.mkdirs(dest);
- }
- }
-
- res = distcp.run(options);
- }
- return res;
-
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java
deleted file mode 100644
index deefbf7..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup.mapreduce;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.IncrementalRestoreService;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.mapreduce.WALPlayer;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class MapReduceRestoreService implements IncrementalRestoreService {
- public static final Log LOG = LogFactory.getLog(MapReduceRestoreService.class);
-
- private WALPlayer player;
-
- public MapReduceRestoreService() {
- this.player = new WALPlayer();
- }
-
- @Override
- public void run(String logDir, String[] tableNames, String[] newTableNames) throws IOException {
- String tableStr = HBackupFileSystem.join(tableNames);
- String newTableStr = HBackupFileSystem.join(newTableNames);
-
- // WALPlayer reads all files in arbitrary directory structure and creates a Map task for each
- // log file
-
- String[] playerArgs = { logDir, tableStr, newTableStr };
- LOG.info("Restore incremental backup from directory " + logDir + " from hbase tables "
- + HBackupFileSystem.join(tableNames) + " to tables "
- + HBackupFileSystem.join(newTableNames));
- try {
- player.run(playerArgs);
- } catch (Exception e) {
- throw new IOException("cannot restore from backup directory " + logDir
- + " (check Hadoop and HBase logs) " + e);
- }
- }
-
- @Override
- public Configuration getConf() {
- return player.getConf();
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.player.setConf(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
deleted file mode 100644
index 4712548..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup.master;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterables;
-
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.backup.BackupSystemTable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
-
-
-
-
-/**
- * Implementation of a log cleaner that checks if a log is still scheduled for
- * incremental backup before deleting it when its TTL is over.
- */
-@InterfaceStability.Evolving
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
-public class BackupLogCleaner extends BaseLogCleanerDelegate {
- private static final Log LOG = LogFactory.getLog(BackupLogCleaner.class);
-
- private boolean stopped = false;
-
- public BackupLogCleaner() {
- }
-
- @Override
- public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
- // all members of this class are null if backup is disabled,
- // so we cannot filter the files
- if (this.getConf() == null) {
- return files;
- }
-
- try {
- final BackupSystemTable table = BackupSystemTable.getTable(getConf());
- // If we do not have recorded backup sessions
- if (table.hasBackupSessions() == false) {
- return files;
- }
- return Iterables.filter(files, new Predicate<FileStatus>() {
- @Override
- public boolean apply(FileStatus file) {
- try {
- String wal = file.getPath().toString();
- boolean logInSystemTable = table.checkWALFile(wal);
- if (LOG.isDebugEnabled()) {
- if (logInSystemTable) {
- LOG.debug("Found log file in hbase:backup, deleting: " + wal);
- } else {
- LOG.debug("Didn't find this log in hbase:backup, keeping: " + wal);
- }
- }
- return logInSystemTable;
- } catch (IOException e) {
- LOG.error(e);
- return false;// keep file for a while, HBase failed
- }
- }
- });
- } catch (IOException e) {
- LOG.error("Failed to get hbase:backup table, therefore will keep all files", e);
- // nothing to delete
- return new ArrayList<FileStatus>();
- }
-
- }
-
- @Override
- public void setConf(Configuration config) {
- // If backup is disabled, keep all members null
- if (!config.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT)) {
- LOG.warn("Backup is disabled - allowing all wals to be deleted");
- return;
- }
- super.setConf(config);
- }
-
- @Override
- public void stop(String why) {
- if (this.stopped) {
- return;
- }
- this.stopped = true;
- LOG.info("Stopping BackupLogCleaner");
- }
-
- @Override
- public boolean isStopped() {
- return this.stopped;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
deleted file mode 100644
index f96682f..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.master;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.MetricsMaster;
-import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
-import org.apache.hadoop.hbase.procedure.Procedure;
-import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
-import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
-import org.apache.zookeeper.KeeperException;
-
-public class LogRollMasterProcedureManager extends MasterProcedureManager {
-
- public static final String ROLLLOG_PROCEDURE_SIGNATURE = "rolllog-proc";
- public static final String ROLLLOG_PROCEDURE_NAME = "rolllog";
- private static final Log LOG = LogFactory.getLog(LogRollMasterProcedureManager.class);
-
- private MasterServices master;
- private ProcedureCoordinator coordinator;
- private boolean done;
-
- @Override
- public void stop(String why) {
- LOG.info("stop: " + why);
- }
-
- @Override
- public boolean isStopped() {
- return false;
- }
-
- @Override
- public void initialize(MasterServices master, MetricsMaster metricsMaster)
- throws KeeperException, IOException, UnsupportedOperationException {
- this.master = master;
- this.done = false;
-
- // setup the default procedure coordinator
- String name = master.getServerName().toString();
- ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, 1);
- BaseCoordinatedStateManager coordManager =
- (BaseCoordinatedStateManager) CoordinatedStateManagerFactory
- .getCoordinatedStateManager(master.getConfiguration());
- coordManager.initialize(master);
-
- ProcedureCoordinatorRpcs comms =
- coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(), name);
-
- this.coordinator = new ProcedureCoordinator(comms, tpool);
- }
-
- @Override
- public String getProcedureSignature() {
- return ROLLLOG_PROCEDURE_SIGNATURE;
- }
-
- @Override
- public void execProcedure(ProcedureDescription desc) throws IOException {
- this.done = false;
- // start the process on the RS
- ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getInstance());
- List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
- List<String> servers = new ArrayList<String>();
- for (ServerName sn : serverNames) {
- servers.add(sn.toString());
- }
- Procedure proc = coordinator.startProcedure(monitor, desc.getInstance(), new byte[0], servers);
- if (proc == null) {
- String msg = "Failed to submit distributed procedure for '" + desc.getInstance() + "'";
- LOG.error(msg);
- throw new IOException(msg);
- }
-
- try {
- // wait for the procedure to complete. A timer thread is kicked off that should cancel this
- // if it takes too long.
- proc.waitForCompleted();
- LOG.info("Done waiting - exec procedure for " + desc.getInstance());
- LOG.info("Distributed roll log procedure is successful!");
- this.done = true;
- } catch (InterruptedException e) {
- ForeignException ee =
- new ForeignException("Interrupted while waiting for roll log procdure to finish", e);
- monitor.receive(ee);
- Thread.currentThread().interrupt();
- } catch (ForeignException e) {
- ForeignException ee =
- new ForeignException("Exception while waiting for roll log procdure to finish", e);
- monitor.receive(ee);
- }
- monitor.rethrowException();
- }
-
- @Override
- public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
- return done;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
deleted file mode 100644
index 618748e..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.regionserver;
-
-import java.util.HashMap;
-import java.util.concurrent.Callable;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.backup.BackupSystemTable;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.procedure.ProcedureMember;
-import org.apache.hadoop.hbase.procedure.Subprocedure;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
-
-
-/**
- * This backup subprocedure implementation forces a log roll on the RS.
- */
-public class LogRollBackupSubprocedure extends Subprocedure {
- private static final Log LOG = LogFactory.getLog(LogRollBackupSubprocedure.class);
-
- private final RegionServerServices rss;
- private final LogRollBackupSubprocedurePool taskManager;
- private FSHLog hlog;
-
- public LogRollBackupSubprocedure(RegionServerServices rss, ProcedureMember member,
- ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout,
- LogRollBackupSubprocedurePool taskManager) {
-
- super(member, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, errorListener,
- wakeFrequency, timeout);
- LOG.info("Constructing a LogRollBackupSubprocedure.");
- this.rss = rss;
- this.taskManager = taskManager;
- }
-
- /**
- * Callable task. TODO. We don't need a thread pool to execute roll log. This can be simplified
- * with no use of subprocedurepool.
- */
- class RSRollLogTask implements Callable<Void> {
- RSRollLogTask() {
- }
-
- @Override
- public Void call() throws Exception {
- if (LOG.isDebugEnabled()) {
- LOG.debug("++ DRPC started: " + rss.getServerName());
- }
- hlog = (FSHLog) rss.getWAL(null);
- long filenum = hlog.getFilenum();
-
- LOG.info("Trying to roll log in backup subprocedure, current log number: " + filenum);
- hlog.rollWriter(true);
- LOG.info("After roll log in backup subprocedure, current log number: " + hlog.getFilenum());
- // write the log number to hbase:backup.
- BackupSystemTable table = BackupSystemTable.getTable(rss.getConfiguration());
- // sanity check, good for testing
- HashMap<String, String> serverTimestampMap = table.readRegionServerLastLogRollResult();
- String host = rss.getServerName().getHostname();
- String sts = serverTimestampMap.get(host);
- if (sts != null && Long.parseLong(sts) > filenum) {
- LOG.warn("Won't update server's last roll log result: current=" + sts + " new=" + filenum);
- return null;
- }
- table.writeRegionServerLastLogRollResult(host, Long.toString(filenum));
- // TODO: potential leak of HBase connection
- // BackupSystemTable.close();
- return null;
- }
-
- }
-
- private void rolllog() throws ForeignException {
-
- monitor.rethrowException();
-
- taskManager.submitTask(new RSRollLogTask());
- monitor.rethrowException();
-
- // wait for everything to complete.
- taskManager.waitForOutstandingTasks();
- monitor.rethrowException();
-
- }
-
- @Override
- public void acquireBarrier() throws ForeignException {
- // do nothing, executing in inside barrier step.
- }
-
- /**
- * do a log roll.
- * @return some bytes
- */
- @Override
- public byte[] insideBarrier() throws ForeignException {
- rolllog();
- // FIXME
- return null;
- }
-
- /**
- * Cancel threads if they haven't finished.
- */
- @Override
- public void cleanup(Exception e) {
- taskManager.abort("Aborting log roll subprocedure tasks for backup due to error", e);
- }
-
- /**
- * Hooray!
- */
- public void releaseBarrier() {
- // NO OP
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java
deleted file mode 100644
index 1ca638c..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.regionserver;
-
-import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.DaemonThreadFactory;
-import org.apache.hadoop.hbase.errorhandling.ForeignException;
-
-/**
- * Handle running each of the individual tasks for completing a backup procedure
- * on a regionserver.
- */
-public class LogRollBackupSubprocedurePool implements Closeable, Abortable {
- private static final Log LOG = LogFactory.getLog(LogRollBackupSubprocedurePool.class);
-
- /** Maximum number of concurrent snapshot region tasks that can run concurrently */
- private static final String CONCURENT_BACKUP_TASKS_KEY = "hbase.backup.region.concurrentTasks";
- private static final int DEFAULT_CONCURRENT_BACKUP_TASKS = 3;
-
- private final ExecutorCompletionService<Void> taskPool;
- private final ThreadPoolExecutor executor;
- private volatile boolean aborted;
- private final List<Future<Void>> futures = new ArrayList<Future<Void>>();
- private final String name;
-
- public LogRollBackupSubprocedurePool(String name, Configuration conf) {
- // configure the executor service
- long keepAlive =
- conf.getLong(LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_KEY,
- LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_DEFAULT);
- int threads = conf.getInt(CONCURENT_BACKUP_TASKS_KEY, DEFAULT_CONCURRENT_BACKUP_TASKS);
- this.name = name;
- executor =
- new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory("rs(" + name
- + ")-backup-pool"));
- taskPool = new ExecutorCompletionService<Void>(executor);
- }
-
- /**
- * Submit a task to the pool.
- */
- public void submitTask(final Callable<Void> task) {
- Future<Void> f = this.taskPool.submit(task);
- futures.add(f);
- }
-
- /**
- * Wait for all of the currently outstanding tasks submitted via {@link #submitTask(Callable)}
- * @return <tt>true</tt> on success, <tt>false</tt> otherwise
- * @throws ForeignException exception
- */
- public boolean waitForOutstandingTasks() throws ForeignException {
- LOG.debug("Waiting for backup procedure to finish.");
-
- try {
- for (Future<Void> f : futures) {
- f.get();
- }
- return true;
- } catch (InterruptedException e) {
- if (aborted) {
- throw new ForeignException("Interrupted and found to be aborted while waiting for tasks!",
- e);
- }
- Thread.currentThread().interrupt();
- } catch (ExecutionException e) {
- if (e.getCause() instanceof ForeignException) {
- throw (ForeignException) e.getCause();
- }
- throw new ForeignException(name, e.getCause());
- } finally {
- // close off remaining tasks
- for (Future<Void> f : futures) {
- if (!f.isDone()) {
- f.cancel(true);
- }
- }
- }
- return false;
- }
-
- /**
- * Attempt to cleanly shutdown any running tasks - allows currently running tasks to cleanly
- * finish
- */
- @Override
- public void close() {
- executor.shutdown();
- }
-
- @Override
- public void abort(String why, Throwable e) {
- if (this.aborted) {
- return;
- }
-
- this.aborted = true;
- LOG.warn("Aborting because: " + why, e);
- this.executor.shutdownNow();
- }
-
- @Override
- public boolean isAborted() {
- return this.aborted;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
deleted file mode 100644
index aca190c..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.regionserver;
-
-
-import java.io.IOException;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.procedure.ProcedureMember;
-import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
-import org.apache.hadoop.hbase.procedure.RegionServerProcedureManager;
-import org.apache.hadoop.hbase.procedure.Subprocedure;
-import org.apache.hadoop.hbase.procedure.SubprocedureFactory;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-
-/**
- * This manager class handles the work dealing with backup for a {@link HRegionServer}.
- * <p>
- * This provides the mechanism necessary to kick off a backup specific {@link Subprocedure} that is
- * responsible by this region server. If any failures occur with the subprocedure, the manager's
- * procedure member notifies the procedure coordinator to abort all others.
- * <p>
- * On startup, requires {@link #start()} to be called.
- * <p>
- * On shutdown, requires org.apache.hadoop.hbase.procedure.ProcedureMember.close() to be
- * called
- */
-public class LogRollRegionServerProcedureManager extends RegionServerProcedureManager {
-
- private static final Log LOG = LogFactory.getLog(LogRollRegionServerProcedureManager.class);
-
- /** Conf key for number of request threads to start backup on regionservers */
- public static final String BACKUP_REQUEST_THREADS_KEY = "hbase.backup.region.pool.threads";
- /** # of threads for backup work on the rs. */
- public static final int BACKUP_REQUEST_THREADS_DEFAULT = 10;
-
- public static final String BACKUP_TIMEOUT_MILLIS_KEY = "hbase.backup.timeout";
- public static final long BACKUP_TIMEOUT_MILLIS_DEFAULT = 60000;
-
- /** Conf key for millis between checks to see if backup work completed or if there are errors */
- public static final String BACKUP_REQUEST_WAKE_MILLIS_KEY = "hbase.backup.region.wakefrequency";
- /** Default amount of time to check for errors while regions finish backup work */
- private static final long BACKUP_REQUEST_WAKE_MILLIS_DEFAULT = 500;
-
- private RegionServerServices rss;
- private ProcedureMemberRpcs memberRpcs;
- private ProcedureMember member;
-
- /**
- * Create a default backup procedure manager
- */
- public LogRollRegionServerProcedureManager() {
- }
-
- /**
- * Start accepting backup procedure requests.
- */
- @Override
- public void start() {
- this.memberRpcs.start(rss.getServerName().toString(), member);
- LOG.info("Started region server backup manager.");
- }
-
- /**
- * Close <tt>this</tt> and all running backup procedure tasks
- * @param force forcefully stop all running tasks
- * @throws IOException exception
- */
- @Override
- public void stop(boolean force) throws IOException {
- String mode = force ? "abruptly" : "gracefully";
- LOG.info("Stopping RegionServerBackupManager " + mode + ".");
-
- try {
- this.member.close();
- } finally {
- this.memberRpcs.close();
- }
- }
-
- /**
- * If in a running state, creates the specified subprocedure for handling a backup procedure.
- * @return Subprocedure to submit to the ProcedureMemeber.
- */
- public Subprocedure buildSubprocedure() {
-
- // don't run a backup if the parent is stop(ping)
- if (rss.isStopping() || rss.isStopped()) {
- throw new IllegalStateException("Can't start backup procedure on RS: " + rss.getServerName()
- + ", because stopping/stopped!");
- }
-
- LOG.info("Attempting to run a roll log procedure for backup.");
- ForeignExceptionDispatcher errorDispatcher = new ForeignExceptionDispatcher();
- Configuration conf = rss.getConfiguration();
- long timeoutMillis = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT);
- long wakeMillis =
- conf.getLong(BACKUP_REQUEST_WAKE_MILLIS_KEY, BACKUP_REQUEST_WAKE_MILLIS_DEFAULT);
-
- LogRollBackupSubprocedurePool taskManager =
- new LogRollBackupSubprocedurePool(rss.getServerName().toString(), conf);
- return new LogRollBackupSubprocedure(rss, member, errorDispatcher, wakeMillis, timeoutMillis,
- taskManager);
-
- }
-
- /**
- * Build the actual backup procedure runner that will do all the 'hard' work
- */
- public class BackupSubprocedureBuilder implements SubprocedureFactory {
-
- @Override
- public Subprocedure buildSubprocedure(String name, byte[] data) {
- return LogRollRegionServerProcedureManager.this.buildSubprocedure();
- }
- }
-
- @Override
- public void initialize(RegionServerServices rss) throws IOException {
- this.rss = rss;
- BaseCoordinatedStateManager coordManager =
- (BaseCoordinatedStateManager) CoordinatedStateManagerFactory.getCoordinatedStateManager(rss
- .getConfiguration());
- coordManager.initialize(rss);
- this.memberRpcs =
- coordManager
- .getProcedureMemberRpcs(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE);
-
- // read in the backup handler configuration properties
- Configuration conf = rss.getConfiguration();
- long keepAlive = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT);
- int opThreads = conf.getInt(BACKUP_REQUEST_THREADS_KEY, BACKUP_REQUEST_THREADS_DEFAULT);
- // create the actual cohort member
- ThreadPoolExecutor pool =
- ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive);
- this.member = new ProcedureMember(memberRpcs, pool, new BackupSubprocedureBuilder());
- }
-
- @Override
- public String getProcedureSignature() {
- return "backup-proc";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
index 3342743..ae36f08 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
@@ -17,11 +17,7 @@
*/
package org.apache.hadoop.hbase.coordination;
-import java.io.IOException;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
-import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.Server;
@@ -55,21 +51,8 @@ public abstract class BaseCoordinatedStateManager implements CoordinatedStateMan
* Method to retrieve coordination for split log worker
*/
public abstract SplitLogWorkerCoordination getSplitLogWorkerCoordination();
-
/**
* Method to retrieve coordination for split log manager
*/
public abstract SplitLogManagerCoordination getSplitLogManagerCoordination();
- /**
- * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs}
- */
- public abstract ProcedureCoordinatorRpcs
- getProcedureCoordinatorRpcs(String procType, String coordNode) throws IOException;
-
- /**
- * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureMemberRpc}
- */
- public abstract ProcedureMemberRpcs
- getProcedureMemberRpcs(String procType) throws IOException;
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
index 7cf4aab..3e89be7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
@@ -17,15 +17,9 @@
*/
package org.apache.hadoop.hbase.coordination;
-import java.io.IOException;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
-import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
-import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
-import org.apache.hadoop.hbase.procedure.ZKProcedureMemberRpcs;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/**
@@ -55,21 +49,9 @@ public class ZkCoordinatedStateManager extends BaseCoordinatedStateManager {
@Override
public SplitLogWorkerCoordination getSplitLogWorkerCoordination() {
return splitLogWorkerCoordination;
- }
-
+ }
@Override
public SplitLogManagerCoordination getSplitLogManagerCoordination() {
return splitLogManagerCoordination;
}
-
- @Override
- public ProcedureCoordinatorRpcs getProcedureCoordinatorRpcs(String procType, String coordNode)
- throws IOException {
- return new ZKProcedureCoordinatorRpcs(watcher, procType, coordNode);
- }
-
- @Override
- public ProcedureMemberRpcs getProcedureMemberRpcs(String procType) throws IOException {
- return new ZKProcedureMemberRpcs(watcher, procType);
- }
}
[11/29] hbase git commit: HBASE-14030 Revert due to pending review
comments
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java
deleted file mode 100644
index f764f18..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java
+++ /dev/null
@@ -1,744 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.concurrent.Callable;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.zookeeper.KeeperException.NoNodeException;
-
-/**
- * A Handler to carry the operations of backup progress
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupHandler implements Callable<Object> {
- private static final Log LOG = LogFactory.getLog(BackupHandler.class);
-
- // backup phase
- // for overall backup (for table list, some table may go online, while some may go offline)
- protected static enum BACKUPPHASE {
- REQUEST, SNAPSHOT, PREPARE_INCREMENTAL, SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST;
- }
-
- // backup status flag
- protected static enum BACKUPSTATUS {
- WAITING, ONGOING, COMPLETE, FAILED, CANCELLED;
- }
-
- protected BackupContext backupContext;
- private BackupManager backupManager;
- private Configuration conf;
-
- public BackupHandler(BackupContext backupContext,
- BackupManager backupManager, Configuration conf) {
- this.backupContext = backupContext;
- this.backupManager = backupManager;
- this.conf = conf;
- }
-
- public BackupContext getBackupContext() {
- return backupContext;
- }
-
- @Override
- public Object call() throws Exception {
- try {
- // overall backup begin
- this.beginBackup(backupContext);
- HashMap<String, String> newTimestamps = null;
- boolean fromExistingSnapshot = false; // supported by future jira
- // handle full or incremental backup for table or table list
- if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
- String savedStartCode = null;
- HBaseAdmin hbadmin = null;
- Connection conn = null;
- boolean firstBackup = false;
- // do snapshot for full table backup, if backing up from existing snapshot, then skip the
- // step of taking snapshot
- if (fromExistingSnapshot) {
- LOG.error("Backup from existing snapshot, so skip the snapshot step. ");
- LOG.error("This feature will be supported by a future jira");
- throw new RuntimeException("Backup from existing snapshot is not supported");
- } else {
- try {
- savedStartCode = backupManager.readBackupStartCode();
- firstBackup = savedStartCode == null;
- if (firstBackup) {
- // This is our first backup. Let's put some marker on ZK so that we can hold the logs
- // while we do the backup.
- backupManager.writeBackupStartCode("0");
- }
- // We roll log here before we do the snapshot. It is possible there is duplicate data
- // in the log that is already in the snapshot. But if we do it after the snapshot, we
- // could have data loss.
- // A better approach is to do the roll log on each RS in the same global procedure as
- // the snapshot.
- LOG.info("Execute roll log procedure for full backup ...");
- conn = ConnectionFactory.createConnection(conf);
- hbadmin = (HBaseAdmin) conn.getAdmin();
- hbadmin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
- LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, new HashMap<String, String>());
- newTimestamps = backupManager.readRegionServerLastLogRollResult();
- if (firstBackup) {
- // Updates registered log files
- // We record ALL old WAL files as registered, because
- // this is a first full backup in the system and these
- // files are not needed for next incremental backup
- List<String> logFiles = BackupUtil.getWALFilesOlderThan(conf, newTimestamps);
- backupManager.recordWALFiles(logFiles);
- }
- this.snapshotForFullBackup(backupContext);
- } catch (BackupException e) {
- // fail the overall backup and return
- this.failBackup(backupContext, e, "Unexpected BackupException : ");
- return null;
- } finally {
- if (hbadmin != null) {
- hbadmin.close();
- }
- if (conn != null) {
- conn.close();
- }
- }
- }
-
- // update the faked progress currently for snapshot done
- this.updateProgress("10.0%", 0);
- // do snapshot copy
- try {
- this.snapshotCopy(backupContext);
- } catch (Exception e) {
- // fail the overall backup and return
- this.failBackup(backupContext, e, "Unexpected BackupException : ");
- return null;
- }
- // Updates incremental backup table set
- backupManager.addIncrementalBackupTableSet(backupContext.getTables());
-
- } else if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- LOG.info("For incremental backup, current table set is "
- + backupManager.getIncrementalBackupTableSet());
- // do incremental table backup preparation
- backupContext.setPhase(BACKUPPHASE.PREPARE_INCREMENTAL);
- // avoid action if has been cancelled
- if (backupContext.isCancelled()) {
- return null;
- }
- try {
- IncrementalBackupManager incrBackupManager = new IncrementalBackupManager(backupManager);
-
- newTimestamps = incrBackupManager.getIncrBackupLogFileList(backupContext);
- } catch (Exception e) {
- // fail the overall backup and return
- this.failBackup(backupContext, e, "Unexpected Exception : ");
- return null;
- }
- // update the faked progress currently for incremental preparation done
- this.updateProgress("10.0%", 0);
-
- // do incremental copy
- try {
- // copy out the table and region info files for each table
- BackupUtil.copyTableRegionInfo(backupContext, conf);
- this.incrementalCopy(backupContext);
- // Save list of WAL files copied
- backupManager.recordWALFiles(backupContext.getIncrBackupFileList());
- } catch (Exception e) {
- // fail the overall backup and return
- this.failBackup(backupContext, e, "Unexpected exception doing incremental copy : ");
- return null;
- }
-
- } else {
- LOG.error("Unsupport backup type: " + backupContext.getType());
- }
-
- // set overall backup status: complete. Here we make sure to complete the backup. After this
- // checkpoint, even if entering cancel process, will let the backup finished
- backupContext.setFlag(BACKUPSTATUS.COMPLETE);
-
- if (!fromExistingSnapshot) {
- if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- // Set the previousTimestampMap which is before this current log roll to the manifest.
- HashMap<String, HashMap<String, String>> previousTimestampMap =
- backupManager.readLogTimestampMap();
- backupContext.setIncrTimestampMap(previousTimestampMap);
- }
- // The table list in backupContext is good for both full backup and incremental backup.
- // For incremental backup, it contains the incremental backup table set.
-
- backupManager.writeRegionServerLogTimestamp(backupContext.getTables(), newTimestamps);
-
- HashMap<String, HashMap<String, String>> newTableSetTimestampMap =
- backupManager.readLogTimestampMap();
-
- String newStartCode =
- BackupUtil.getMinValue(BackupUtil.getRSLogTimestampMins(newTableSetTimestampMap));
- backupManager.writeBackupStartCode(newStartCode);
- }
-
- // backup complete
- this.completeBackup(backupContext);
- } catch (Exception e) {
- // even during completing backup (#completeBackup(backupContext)), exception may occur, or
- // exception occur during other process, fail the backup finally
- this.failBackup(backupContext, e, "Error caught during backup progress: ");
- }
- return null;
- }
-
- /**
- * Begin the overall backup.
- * @param backupContext backup context
- * @throws IOException exception
- */
- private void beginBackup(BackupContext backupContext) throws IOException {
-
- // set the start timestamp of the overall backup
- long startTs = EnvironmentEdgeManager.currentTime();
- backupContext.setStartTs(startTs);
- // set overall backup status: ongoing
- backupContext.setFlag(BACKUPSTATUS.ONGOING);
- LOG.info("Backup " + backupContext.getBackupId() + " starts at " + startTs + ".");
-
- backupManager.updateBackupStatus(backupContext);
- LOG.debug("Backup session " + backupContext.getBackupId() + " has been started.");
-
- }
-
- /**
- * Snapshot for full table backup.
- * @param backupContext backup context
- * @throws IOException exception
- */
- private void snapshotForFullBackup(BackupContext backupContext) throws IOException {
-
- LOG.info("HBase snapshot full backup for " + backupContext.getBackupId());
-
- // avoid action if has been cancelled
- if (backupContext.isCancelled()) {
- return;
- }
-
- HBaseAdmin hbadmin = null;
- Connection conn = null;
-
- // we do HBase snapshot for tables in the table list one by one currently
- for (String table : backupContext.getTables()) {
-
- // avoid action if it has been cancelled
- if (backupContext.isCancelled()) {
- return;
- }
-
- HBaseProtos.SnapshotDescription backupSnapshot;
- try {
- // wrap a SnapshotDescription for offline/online snapshot
- backupSnapshot = this.wrapSnapshotDescription(table);
-
- // set the snapshot name in BackupStatus of this table
- backupContext.setSnapshotName(table, backupSnapshot.getName());
-
- // Kick off snapshot for backup
- conn = ConnectionFactory.createConnection(conf);
- hbadmin = (HBaseAdmin) conn.getAdmin();
- hbadmin.snapshot(backupSnapshot);
-
- if (LOG.isDebugEnabled() == false) {
- // In DEBUG mode we log message already.
- // This is not to duplicate that message.
- LOG.info("Snapshot has been launched, waiting to finish ...");
- }
-
- } catch (Exception e) {
- LOG.error("Snapshot failed to create " + getMessage(e));
-
- // currently, we fail the overall backup if any table in the list failed, so throw the
- // exception out for overall backup failing
- throw new BackupException("Backup snapshot failed on table " + table, e);
- } finally {
- if (hbadmin != null) {
- hbadmin.close();
- }
- if (conn != null) {
- conn.close();
- }
- }
-
- // set the snapshot name in BackupStatus of this table, only after snapshot success.
- backupContext.setSnapshotName(table, backupSnapshot.getName());
-
- } // for each table in the backup table list
-
- }
-
- /**
- * Fail the overall backup.
- * @param backupContext backup context
- * @param e exception
- * @throws Exception exception
- */
- private void failBackup(BackupContext backupContext, Exception e, String msg) throws Exception {
-
- LOG.error(msg + getMessage(e));
-
- // If this is a cancel exception, then we've already cleaned.
-
- if (this.backupContext.getFlag().equals(BACKUPSTATUS.CANCELLED)) {
- return;
- }
-
- // set the failure timestamp of the overall backup
- backupContext.setEndTs(EnvironmentEdgeManager.currentTime());
-
- // set failure message
- backupContext.setFailedMsg(e.getMessage());
-
- // set overall backup status: failed
- backupContext.setFlag(BACKUPSTATUS.FAILED);
-
- // compose the backup failed data
- String backupFailedData =
- "BackupId=" + backupContext.getBackupId() + ",startts=" + backupContext.getStartTs()
- + ",failedts=" + backupContext.getEndTs() + ",failedphase=" + backupContext.getPhase()
- + ",failedmessage=" + backupContext.getFailedMsg();
- LOG.error(backupFailedData);
-
- backupManager.updateBackupStatus(backupContext);
-
- // if full backup, then delete HBase snapshots if there already have snapshots taken
- // and also clean up export snapshot log files if exist
- if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
- if (!backupContext.fromExistingSnapshot()) {
- this.deleteSnapshot(backupContext);
- }
- this.cleanupExportSnapshotLog();
- } /*
- * else { // support incremental backup code in future jira // TODO. See HBASE-14124 }
- */
-
- // clean up the uncompleted data at target directory if the ongoing backup has already entered
- // the copy phase
- // For incremental backup, DistCp logs will be cleaned with the targetDir.
- this.cleanupTargetDir();
-
- LOG.info("Backup " + backupContext.getBackupId() + " failed.");
- }
-
- /**
- * Update the ongoing back token znode with new progress.
- * @param newProgress progress
- * @param bytesCopied bytes copied
- * @throws NoNodeException exception
- */
- public void updateProgress(String newProgress, long bytesCopied) throws IOException {
-
- // compose the new backup progress data, using fake number for now
- String backupProgressData = newProgress;
-
- backupContext.setProgress(newProgress);
- backupManager.updateBackupStatus(backupContext);
- LOG.debug("Backup progress data \"" + backupProgressData
- + "\" has been updated to hbase:backup for " + backupContext.getBackupId());
- }
-
- /**
- * Complete the overall backup.
- * @param backupContext backup context
- * @throws Exception exception
- */
- private void completeBackup(BackupContext backupContext) throws Exception {
-
- // set the complete timestamp of the overall backup
- backupContext.setEndTs(EnvironmentEdgeManager.currentTime());
- // set overall backup status: complete
- backupContext.setFlag(BACKUPSTATUS.COMPLETE);
- // add and store the manifest for the backup
- this.addManifest(backupContext);
-
- // after major steps done and manifest persisted, do convert if needed for incremental backup
- /* in-fly convert code here, provided by future jira */
- LOG.debug("in-fly convert code here, provided by future jira");
-
- // compose the backup complete data
- String backupCompleteData =
- this.obtainBackupMetaDataStr(backupContext) + ",startts=" + backupContext.getStartTs()
- + ",completets=" + backupContext.getEndTs() + ",bytescopied="
- + backupContext.getTotalBytesCopied();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Backup " + backupContext.getBackupId() + " finished: " + backupCompleteData);
- }
- backupManager.updateBackupStatus(backupContext);
-
- // when full backup is done:
- // - delete HBase snapshot
- // - clean up directories with prefix "exportSnapshot-", which are generated when exporting
- // snapshots
- if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
- if (!backupContext.fromExistingSnapshot()) {
- this.deleteSnapshot(backupContext);
- }
- this.cleanupExportSnapshotLog();
- } else if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- this.cleanupDistCpLog();
- } else {
- LOG.error(" other backup types have not been implemented yet");
- }
-
- LOG.info("Backup " + backupContext.getBackupId() + " completed.");
- }
-
- /**
- * Get backup request meta data dir as string.
- * @param backupContext backup context
- * @return meta data dir
- */
- private String obtainBackupMetaDataStr(BackupContext backupContext) {
- StringBuffer sb = new StringBuffer();
- sb.append("type=" + backupContext.getType() + ",tablelist=");
- for (String table : backupContext.getTables()) {
- sb.append(table + ";");
- }
- if (sb.lastIndexOf(";") > 0) {
- sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1);
- }
- sb.append(",targetRootDir=" + backupContext.getTargetRootDir());
- if (backupContext.fromExistingSnapshot()) {
- sb.append(",snapshot=" + backupContext.getExistingSnapshot());
- }
- return sb.toString();
- }
-
- /**
- * Do snapshot copy.
- * @param backupContext backup context
- * @throws Exception exception
- */
- private void snapshotCopy(BackupContext backupContext) throws Exception {
-
- LOG.info("Snapshot copy is starting.");
-
- // set overall backup phase: snapshot_copy
- backupContext.setPhase(BACKUPPHASE.SNAPSHOTCOPY);
-
- // avoid action if has been cancelled
- if (backupContext.isCancelled()) {
- return;
- }
-
- // call ExportSnapshot to copy files based on hbase snapshot for backup
- // ExportSnapshot only support single snapshot export, need loop for multiple tables case
- BackupCopyService copyService = BackupRestoreServiceFactory.getBackupCopyService(conf);
-
- // number of snapshots matches number of tables
- float numOfSnapshots = backupContext.getSnapshotNames().size();
-
- LOG.debug("There are " + (int) numOfSnapshots + " snapshots to be copied.");
-
- for (String table : backupContext.getTables()) {
- // Currently we simply set the sub copy tasks by counting the table snapshot number, we can
- // calculate the real files' size for the percentage in the future.
- // TODO this below
- // backupCopier.setSubTaskPercntgInWholeTask(1f / numOfSnapshots);
- int res = 0;
- String[] args = new String[4];
- args[0] = "-snapshot";
- args[1] = backupContext.getSnapshotName(table);
- args[2] = "-copy-to";
- args[3] = backupContext.getBackupStatus(table).getTargetDir();
-
- LOG.debug("Copy snapshot " + args[1] + " to " + args[3]);
- res = copyService.copy(this, conf, BackupCopyService.Type.FULL, args);
- // if one snapshot export failed, do not continue for remained snapshots
- if (res != 0) {
-
- LOG.error("Exporting Snapshot " + args[1] + " failed with return code: " + res + ".");
-
- throw new IOException("Failed of exporting snapshot " + args[1] + " to " + args[3]
- + " with reason code " + res);
- }
-
- LOG.info("Snapshot copy " + args[1] + " finished.");
- }
- }
-
- /**
- * Wrap a SnapshotDescription for a target table.
- * @param table table
- * @return a SnapshotDescription especially for backup.
- */
- private SnapshotDescription wrapSnapshotDescription(String table) {
- // Mock a SnapshotDescription from backupContext to call SnapshotManager function,
- // Name it in the format "snapshot_<timestamp>_<table>"
- HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder();
- builder.setTable(table);
- TableName tableName = TableName.valueOf(table);
- builder.setName("snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime()) + "_"
- + tableName.getNamespaceAsString() + "_" + tableName.getQualifierAsString());
- HBaseProtos.SnapshotDescription backupSnapshot = builder.build();
-
- LOG.debug("Wrapped a SnapshotDescription " + backupSnapshot.getName()
- + " from backupContext to request snapshot for backup.");
-
- return backupSnapshot;
- }
-
- /**
- * Delete HBase snapshot for backup.
- * @param backupCtx backup context
- * @throws Exception exception
- */
- private void deleteSnapshot(BackupContext backupCtx) throws IOException {
-
- LOG.debug("Trying to delete snapshot for full backup.");
- Connection conn = null;
- Admin admin = null;
- try {
- conn = ConnectionFactory.createConnection(conf);
- admin = conn.getAdmin();
- for (String snapshotName : backupCtx.getSnapshotNames()) {
- if (snapshotName == null) {
- continue;
- }
- LOG.debug("Trying to delete snapshot: " + snapshotName);
- admin.deleteSnapshot(snapshotName);
- LOG.debug("Deleting the snapshot " + snapshotName + " for backup "
- + backupCtx.getBackupId() + " succeeded.");
- }
- } finally {
- if (admin != null) {
- admin.close();
- }
- if (conn != null) {
- conn.close();
- }
- }
- }
-
- /**
- * Clean up directories with prefix "exportSnapshot-", which are generated when exporting
- * snapshots.
- * @throws IOException exception
- */
- private void cleanupExportSnapshotLog() throws IOException {
- FileSystem fs = FSUtils.getCurrentFileSystem(conf);
- Path stagingDir =
- new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory()
- .toString()));
- FileStatus[] files = FSUtils.listStatus(fs, stagingDir);
- if (files == null) {
- return;
- }
- for (FileStatus file : files) {
- if (file.getPath().getName().startsWith("exportSnapshot-")) {
- LOG.debug("Delete log files of exporting snapshot: " + file.getPath().getName());
- if (FSUtils.delete(fs, file.getPath(), true) == false) {
- LOG.warn("Can not delete " + file.getPath());
- }
- }
- }
- }
-
- /**
- * Clean up directories with prefix "_distcp_logs-", which are generated when DistCp copying
- * hlogs.
- * @throws IOException exception
- */
- private void cleanupDistCpLog() throws IOException {
- Path rootPath = new Path(backupContext.getHLogTargetDir()).getParent();
- FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
- FileStatus[] files = FSUtils.listStatus(fs, rootPath);
- if (files == null) {
- return;
- }
- for (FileStatus file : files) {
- if (file.getPath().getName().startsWith("_distcp_logs")) {
- LOG.debug("Delete log files of DistCp: " + file.getPath().getName());
- FSUtils.delete(fs, file.getPath(), true);
- }
- }
- }
-
- /**
- * Clean up the uncompleted data at target directory if the ongoing backup has already entered the
- * copy phase.
- */
- private void cleanupTargetDir() {
- try {
- // clean up the uncompleted data at target directory if the ongoing backup has already entered
- // the copy phase
- LOG.debug("Trying to cleanup up target dir. Current backup phase: "
- + backupContext.getPhase());
- if (backupContext.getPhase().equals(BACKUPPHASE.SNAPSHOTCOPY)
- || backupContext.getPhase().equals(BACKUPPHASE.INCREMENTAL_COPY)
- || backupContext.getPhase().equals(BACKUPPHASE.STORE_MANIFEST)) {
- FileSystem outputFs =
- FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf);
-
- // now treat one backup as a transaction, clean up data that has been partially copied at
- // table level
- for (String table : backupContext.getTables()) {
- Path targetDirPath =
- new Path(HBackupFileSystem.getTableBackupDir(backupContext.getTargetRootDir(),
- backupContext.getBackupId(), table));
- if (outputFs.delete(targetDirPath, true)) {
- LOG.info("Cleaning up uncompleted backup data at " + targetDirPath.toString()
- + " done.");
- } else {
- LOG.info("No data has been copied to " + targetDirPath.toString() + ".");
- }
-
- Path tableDir = targetDirPath.getParent();
- FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir);
- if (backups == null || backups.length == 0) {
- outputFs.delete(tableDir, true);
- LOG.debug(tableDir.toString() + " is empty, remove it.");
- }
- }
- }
-
- } catch (IOException e1) {
- LOG.error("Cleaning up uncompleted backup data of " + backupContext.getBackupId() + " at "
- + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
- }
- }
-
- /**
- * Add manifest for the current backup. The manifest is stored
- * within the table backup directory.
- * @param backupContext The current backup context
- * @throws IOException exception
- * @throws BackupException exception
- */
- private void addManifest(BackupContext backupContext) throws IOException, BackupException {
- // set the overall backup phase : store manifest
- backupContext.setPhase(BACKUPPHASE.STORE_MANIFEST);
-
- // avoid action if has been cancelled
- if (backupContext.isCancelled()) {
- return;
- }
-
- BackupManifest manifest;
- boolean fromExistingSnapshot = false; // to be implemented in future jira
-
- // Since we have each table's backup in its own directory structure,
- // we'll store its manifest with the table directory.
- for (String table : backupContext.getTables()) {
- manifest = new BackupManifest(backupContext, table);
- if (fromExistingSnapshot) {
- // mark backing up from existing snapshot in manifest, so that later, dependency analysis
- // can skip this backup image
- LOG.debug("backup using existing snapshot will be supported in future jira");
- } else {
- ArrayList<BackupImage> ancestors = this.backupManager.getAncestors(backupContext, table);
- for (BackupImage image : ancestors) {
- manifest.addDependentImage(image);
- }
- }
- if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- // We'll store the log timestamps for this table only in its manifest.
- HashMap<String, HashMap<String, String>> tableTimestampMap =
- new HashMap<String, HashMap<String, String>>();
- tableTimestampMap.put(table, backupContext.getIncrTimestampMap().get(table));
- manifest.setIncrTimestampMap(tableTimestampMap);
- }
- manifest.store(conf);
- }
-
- // For incremental backup, we store a overall manifest in
- // <backup-root-dir>/WALs/<backup-id>
- // This is used when created the next incremental backup
- if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- manifest = new BackupManifest(backupContext);
- // set the table region server start and end timestamps for incremental backup
- manifest.setIncrTimestampMap(backupContext.getIncrTimestampMap());
- ArrayList<BackupImage> ancestors = this.backupManager.getAncestors(backupContext);
- for (BackupImage image : ancestors) {
- manifest.addDependentImage(image);
- }
- manifest.store(conf);
- }
- }
-
- /**
- * Do incremental copy.
- * @param backupContext backup context
- */
- private void incrementalCopy(BackupContext backupContext) throws Exception {
-
- LOG.info("Incremental copy is starting.");
-
- // set overall backup phase: incremental_copy
- backupContext.setPhase(BACKUPPHASE.INCREMENTAL_COPY);
-
- // avoid action if has been cancelled
- if (backupContext.isCancelled()) {
- return;
- }
-
- // get incremental backup file list and prepare parms for DistCp
- List<String> incrBackupFileList = backupContext.getIncrBackupFileList();
- String[] strArr = incrBackupFileList.toArray(new String[incrBackupFileList.size() + 1]);
- strArr[strArr.length - 1] = backupContext.getHLogTargetDir();
-
- BackupCopyService copyService = BackupRestoreServiceFactory.getBackupCopyService(conf);
- int res = copyService.copy(this, conf, BackupCopyService.Type.INCREMENTAL, strArr);
-
- if (res != 0) {
- LOG.error("Copy incremental log files failed with return code: " + res + ".");
- throw new IOException("Failed of Hadoop Distributed Copy from " + incrBackupFileList + " to "
- + backupContext.getHLogTargetDir());
- }
- LOG.info("Incremental copy from " + incrBackupFileList + " to "
- + backupContext.getHLogTargetDir() + " finished.");
-
- }
-
- private String getMessage(Exception e) {
- String msg = e.getMessage();
- if (msg == null || msg.equals("")) {
- msg = e.getClass().getName();
- }
- return msg;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java
deleted file mode 100644
index fdb3c46..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
-import org.apache.hadoop.hbase.backup.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData;
-import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-
-
-
-/**
- * Handles backup requests on server-side, creates backup context records in hbase:backup
- * to keep track backup. The timestamps kept in hbase:backup table will be used for future
- * incremental backup. Creates BackupContext and DispatchRequest.
- *
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupManager {
- private static final Log LOG = LogFactory.getLog(BackupManager.class);
-
- private Configuration conf = null;
- private BackupContext backupContext = null;
- private ExecutorService pool = null;
-
- private boolean backupComplete = false;
-
- private BackupSystemTable systemTable;
-
- /**
- * Backup manager constructor.
- * @param conf configuration
- * @throws IOException exception
- */
- public BackupManager(Configuration conf) throws IOException {
- if (!conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT)) {
- throw new BackupException("HBase backup is not enabled. Check your " +
- HConstants.BACKUP_ENABLE_KEY + " setting.");
- }
- this.conf = conf;
- this.systemTable = BackupSystemTable.getTable(conf);
- Runtime.getRuntime().addShutdownHook(new ExitHandler());
- }
-
- /**
- * This method modifies the master's configuration in order to inject backup-related features
- * @param conf configuration
- */
- public static void decorateMasterConfiguration(Configuration conf) {
- if (!isBackupEnabled(conf)) {
- return;
- }
- String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
- String cleanerClass = BackupLogCleaner.class.getCanonicalName();
- if (!plugins.contains(cleanerClass)) {
- conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added log cleaner: " + cleanerClass);
- }
- }
-
- private static boolean isBackupEnabled(Configuration conf) {
- return conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT);
- }
-
- private class ExitHandler extends Thread {
- public ExitHandler() {
- super("Backup Manager Exit Handler");
- }
-
- public void run() {
-
- if (backupContext != null && !backupComplete) {
-
- // program exit and backup is not complete, then mark as cancelled to avoid submitted backup
- // handler's taking further action
- backupContext.markCancel();
-
- LOG.debug("Backup is cancelled due to force program exiting.");
- try {
- cancelBackup(backupContext.getBackupId());
- } catch (Exception e) {
- String msg = e.getMessage();
- if (msg == null || msg.equals("")) {
- msg = e.getClass().getName();
- }
- LOG.error("Failed to cancel backup " + backupContext.getBackupId() + " due to " + msg);
- }
- }
-
- exit();
- }
- }
-
- /**
- * Cancel the ongoing backup via backup id.
- * @param backupId The id of the ongoing backup to be cancelled
- * @throws Exception exception
- */
- private void cancelBackup(String backupId) throws Exception {
- // TODO: will be implemented in Phase 2: HBASE-14125
- LOG.debug("Try to cancel the backup " + backupId + ". the feature is NOT implemented yet");
-
- }
-
- /**
- * Stop all the work of backup.
- */
- public void exit() {
-
- // currently, we shutdown now for all ongoing back handlers, we may need to do something like
- // record the failed list somewhere later
- if (this.pool != null) {
- this.pool.shutdownNow();
- }
-
- }
-
- /**
- * Create a BackupContext based on input backup request.
- * @param backupId backup id
- * @param type type
- * @param tablelist table list
- * @param targetRootDir root dir
- * @param snapshot snapshot name
- * @return BackupContext context
- * @throws BackupException exception
- */
- protected BackupContext createBackupContext(String backupId, String type, String tablelist,
- String targetRootDir, String snapshot) throws BackupException {
-
- if (targetRootDir == null) {
- throw new BackupException("Wrong backup request parameter: target backup root directory");
- }
-
- if (type.equals(BackupRestoreConstants.BACKUP_TYPE_FULL) && tablelist == null) {
- // If table list is null for full backup, which means backup all tables. Then fill the table
- // list with all user tables from meta. It no table available, throw the request exception.
-
- HTableDescriptor[] htds = null;
- try (Connection conn = ConnectionFactory.createConnection(conf);
- HBaseAdmin hbadmin = (HBaseAdmin) conn.getAdmin()) {
-
- htds = hbadmin.listTables();
- } catch (Exception e) {
- throw new BackupException(e);
- }
-
- if (htds == null) {
- throw new BackupException("No table exists for full backup of all tables.");
- } else {
- StringBuilder sb = new StringBuilder();
- for (HTableDescriptor hTableDescriptor : htds) {
- sb.append(hTableDescriptor.getNameAsString()
- + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- }
- sb.deleteCharAt(sb.lastIndexOf(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND));
- tablelist = sb.toString();
-
- LOG.info("Full backup all the tables available in the cluster: " + tablelist);
- }
- }
-
- // there are one or more tables in the table list
- return new BackupContext(backupId, type,
- tablelist.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND), targetRootDir,
- snapshot);
-
- }
-
- /**
- * Check if any ongoing backup. Currently, we only reply on checking status in hbase:backup. We
- * need to consider to handle the case of orphan records in the future. Otherwise, all the coming
- * request will fail.
- * @return the ongoing backup id if on going backup exists, otherwise null
- * @throws IOException exception
- */
- private String getOngoingBackupId() throws IOException {
-
- ArrayList<BackupContext> sessions = systemTable.getBackupContexts(BACKUPSTATUS.ONGOING);
- if (sessions.size() == 0) {
- return null;
- }
- return sessions.get(0).getBackupId();
- }
-
- /**
- * Start the backup manager service.
- * @throws IOException exception
- */
- public void initialize() throws IOException {
- String ongoingBackupId = this.getOngoingBackupId();
- if (ongoingBackupId != null) {
- LOG.info("There is a ongoing backup " + ongoingBackupId
- + ". Can not launch new backup until no ongoing backup remains.");
- throw new BackupException("There is ongoing backup.");
- }
-
- // Initialize thread pools
- int nrThreads = this.conf.getInt("hbase.backup.threads.max", 1);
- ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
- builder.setNameFormat("BackupHandler-%1$d");
- this.pool =
- new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), builder.build());
- ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
- }
-
- /**
- * Dispatch and handle a backup request.
- * @param backupContext backup context
- * @throws BackupException exception
- */
- public void dispatchRequest(BackupContext backupContext) throws BackupException {
-
- this.backupContext = backupContext;
-
- LOG.info("Got a backup request: " + "Type: " + backupContext.getType() + "; Tables: "
- + backupContext.getTableListAsString() + "; TargetRootDir: "
- + backupContext.getTargetRootDir());
-
- // dispatch the request to a backup handler and put it handler map
-
- BackupHandler handler = new BackupHandler(this.backupContext, this, conf);
- Future<Object> future = this.pool.submit(handler);
- // wait for the execution to complete
- try {
- future.get();
- } catch (InterruptedException e) {
- throw new BackupException(e);
- } catch (CancellationException e) {
- throw new BackupException(e);
- } catch (ExecutionException e) {
- throw new BackupException(e);
- }
-
- // mark the backup complete for exit handler's processing
- backupComplete = true;
-
- LOG.info("Backup request " + backupContext.getBackupId() + " has been executed.");
- }
-
- /**
- * Get direct ancestors of the current backup.
- * @param backupCtx The backup context for the current backup
- * @return The ancestors for the current backup
- * @throws IOException exception
- * @throws BackupException exception
- */
- protected ArrayList<BackupImage> getAncestors(BackupContext backupCtx) throws IOException,
- BackupException {
- LOG.debug("Getting the direct ancestors of the current backup ...");
-
- ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>();
-
- // full backup does not have ancestor
- if (backupCtx.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
- LOG.debug("Current backup is a full backup, no direct ancestor for it.");
- return ancestors;
- }
-
- // get all backup history list in descending order
-
- ArrayList<BackupCompleteData> allHistoryList = getBackupHistory();
- for (BackupCompleteData backup : allHistoryList) {
- BackupImage image =
- new BackupImage(backup.getBackupToken(), backup.getType(), backup.getBackupRootPath(),
- backup.getTableList(), Long.parseLong(backup.getStartTime()), Long.parseLong(backup
- .getEndTime()));
-
- // add the full backup image as an ancestor until the last incremental backup
- if (backup.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
-
- // backup image from existing snapshot does not involve in dependency
- if (backup.fromExistingSnapshot()) {
- continue;
- }
- // check the backup image coverage, if previous image could be covered by the newer ones,
- // then no need to add
- if (!BackupManifest.canCoverImage(ancestors, image)) {
- ancestors.add(image);
- }
- } else {
- // found last incremental backup, if previously added full backup ancestor images can cover
- // it, then this incremental ancestor is not the dependent of the current incremental
- // backup, that is to say, this is the backup scope boundary of current table set.
- // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing
- // incremental backup
- if (BackupManifest.canCoverImage(ancestors, image)) {
- LOG.debug("Met the backup boundary of the current table set. "
- + "The root full backup images for the current backup scope:");
- for (BackupImage image1 : ancestors) {
- LOG.debug(" BackupId: " + image1.getBackupId() + ", Backup directory: "
- + image1.getRootDir());
- }
- } else {
- Path logBackupPath =
- HBackupFileSystem.getLogBackupPath(backup.getBackupRootPath(),
- backup.getBackupToken());
- LOG.debug("Current backup has an incremental backup ancestor, "
- + "touching its image manifest in " + logBackupPath.toString()
- + " to construct the dependency.");
-
- BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath);
- BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage();
- ancestors.add(lastIncrImage);
-
- LOG.debug("Last dependent incremental backup image information:");
- LOG.debug(" Token: " + lastIncrImage.getBackupId());
- LOG.debug(" Backup directory: " + lastIncrImage.getRootDir());
- }
- }
- }
- LOG.debug("Got " + ancestors.size() + " ancestors for the current backup.");
- return ancestors;
- }
-
- /**
- * Get the direct ancestors of this backup for one table involved.
- * @param backupContext backup context
- * @param table table
- * @return backupImages on the dependency list
- * @throws BackupException exception
- * @throws IOException exception
- */
- protected ArrayList<BackupImage> getAncestors(BackupContext backupContext, String table)
- throws BackupException, IOException {
- ArrayList<BackupImage> ancestors = getAncestors(backupContext);
- ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>();
- for (BackupImage image : ancestors) {
- if (image.hasTable(table)) {
- tableAncestors.add(image);
- if (image.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
- break;
- }
- }
- }
- return tableAncestors;
- }
-
- /**
- * hbase:backup operations
- */
-
- /**
- * Updates status (state) of a backup session in a persistent store
- * @param context context
- * @throws IOException exception
- */
- public void updateBackupStatus(BackupContext context) throws IOException {
- systemTable.updateBackupStatus(context);
- }
-
- /**
- * Read the last backup start code (timestamp) of last successful backup. Will return null
- * if there is no startcode stored in hbase:backup or the value is of length 0. These two
- * cases indicate there is no successful backup completed so far.
- * @return the timestamp of a last successful backup
- * @throws IOException exception
- */
- public String readBackupStartCode() throws IOException {
- return systemTable.readBackupStartCode();
- }
-
- /**
- * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte.
- * @param startCode start code
- * @throws IOException exception
- */
- public void writeBackupStartCode(String startCode) throws IOException {
- systemTable.writeBackupStartCode(startCode);
- }
-
- /**
- * Get the RS log information after the last log roll from hbase:backup.
- * @return RS log info
- * @throws IOException exception
- */
- public HashMap<String, String> readRegionServerLastLogRollResult() throws IOException {
- return systemTable.readRegionServerLastLogRollResult();
- }
-
- /**
- * Get all completed backup information (in desc order by time)
- * @return history info of BackupCompleteData
- * @throws IOException exception
- */
- public ArrayList<BackupCompleteData> getBackupHistory() throws IOException {
- return systemTable.getBackupHistory();
- }
-
- /**
- * Write the current timestamps for each regionserver to hbase:backup after a successful full or
- * incremental backup. Each table may have a different set of log timestamps. The saved timestamp
- * is of the last log file that was backed up already.
- * @param tables tables
- * @throws IOException exception
- */
- public void writeRegionServerLogTimestamp(Set<String> tables,
- HashMap<String, String> newTimestamps) throws IOException {
- systemTable.writeRegionServerLogTimestamp(tables, newTimestamps);
- }
-
- /**
- * Read the timestamp for each region server log after the last successful backup. Each table has
- * its own set of the timestamps. The info is stored for each table as a concatinated string on ZK
- * under //hbase//backup//incr//tablelogtimestamp//table_name
- * @return the timestamp for each region server. key: tableName value:
- * RegionServer,PreviousTimeStamp
- * @throws IOException exception
- */
- public HashMap<String, HashMap<String, String>> readLogTimestampMap() throws IOException {
- return systemTable.readLogTimestampMap();
- }
-
- /**
- * Return the current tables covered by incremental backup.
- * @return set of tableNames
- * @throws IOException exception
- */
- public Set<String> getIncrementalBackupTableSet() throws IOException {
- return systemTable.getIncrementalBackupTableSet();
- }
-
- /**
- * Adds set of tables to overall incremental backup table set
- * @param tables tables
- * @throws IOException exception
- */
- public void addIncrementalBackupTableSet(Set<String> tables) throws IOException {
- systemTable.addIncrementalBackupTableSet(tables);
- }
-
- /**
- * Saves list of WAL files after incremental backup operation. These files will be stored until
- * TTL expiration and are used by Backup Log Cleaner plugin to determine which WAL files can be
- * safely purged.
- */
-
- public void recordWALFiles(List<String> files) throws IOException {
- systemTable.addWALFiles(files, backupContext.getBackupId());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java
deleted file mode 100644
index f41540b..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java
+++ /dev/null
@@ -1,814 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-
-/**
- * Backup manifest Contains all the meta data of a backup image. The manifest info will be bundled
- * as manifest file together with data. So that each backup image will contain all the info needed
- * for restore.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupManifest {
-
- private static final Log LOG = LogFactory.getLog(BackupManifest.class);
-
- // manifest file name
- public static final String FILE_NAME = ".backup.manifest";
-
- // manifest file version, current is 1.0
- public static final String MANIFEST_VERSION = "1.0";
-
- // tags of fields for manifest file
- public static final String TAG_VERSION = "Manifest-Version";
- public static final String TAG_BACKUPID = "Backup-Id";
- public static final String TAG_BACKUPTYPE = "Backup-Type";
- public static final String TAG_TABLESET = "Table-Set";
- public static final String TAG_STARTTS = "Start-Timestamp";
- public static final String TAG_COMPLETETS = "Complete-Timestamp";
- public static final String TAG_TABLEBYTES = "Total-Table-Bytes";
- public static final String TAG_LOGBYTES = "Total-Log-Bytes";
- public static final String TAG_INCRTIMERANGE = "Incremental-Time-Range";
- public static final String TAG_DEPENDENCY = "Dependency";
- public static final String TAG_IMAGESTATE = "Image-State";
- public static final String TAG_COMPACTION = "Compaction";
-
- public static final String ERROR_DEPENDENCY = "DEPENDENCY_ERROR";
-
- public static final int DELETE_SUCCESS = 0;
- public static final int DELETE_FAILED = -1;
-
- // currently only one state, will have CONVERTED, and MERGED in future JIRA
- public static final String IMAGE_STATE_ORIG = "ORIGINAL";
- public static final String IMAGE_STATE_CONVERT = "CONVERTED";
- public static final String IMAGE_STATE_MERGE = "MERGED";
- public static final String IMAGE_STATE_CONVERT_MERGE = "CONVERTED,MERGED";
-
- // backup image, the dependency graph is made up by series of backup images
-
- public static class BackupImage implements Comparable<BackupImage> {
-
- private String backupId;
- private String type;
- private String rootDir;
- private String tableSet;
- private long startTs;
- private long completeTs;
- private ArrayList<BackupImage> ancestors;
-
- public BackupImage() {
- super();
- }
-
- public BackupImage(String backupId, String type, String rootDir, String tableSet, long startTs,
- long completeTs) {
- this.backupId = backupId;
- this.type = type;
- this.rootDir = rootDir;
- this.tableSet = tableSet;
- this.startTs = startTs;
- this.completeTs = completeTs;
- }
-
- public String getBackupId() {
- return backupId;
- }
-
- public void setBackupId(String backupId) {
- this.backupId = backupId;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getRootDir() {
- return rootDir;
- }
-
- public void setRootDir(String rootDir) {
- this.rootDir = rootDir;
- }
-
- public String getTableSet() {
- return tableSet;
- }
-
- public void setTableSet(String tableSet) {
- this.tableSet = tableSet;
- }
-
- public long getStartTs() {
- return startTs;
- }
-
- public void setStartTs(long startTs) {
- this.startTs = startTs;
- }
-
- public long getCompleteTs() {
- return completeTs;
- }
-
- public void setCompleteTs(long completeTs) {
- this.completeTs = completeTs;
- }
-
- public ArrayList<BackupImage> getAncestors() {
- if (this.ancestors == null) {
- this.ancestors = new ArrayList<BackupImage>();
- }
- return this.ancestors;
- }
-
- public void addAncestor(BackupImage backupImage) {
- this.getAncestors().add(backupImage);
- }
-
- public boolean hasAncestor(String token) {
- for (BackupImage image : this.getAncestors()) {
- if (image.getBackupId().equals(token)) {
- return true;
- }
- }
- return false;
- }
-
- public boolean hasTable(String table) {
- String[] tables = this.getTableSet().split(";");
- for (String t : tables) {
- if (t.equals(table)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public int compareTo(BackupImage other) {
- String thisBackupId = this.getBackupId();
- String otherBackupId = other.getBackupId();
- Long thisTS = new Long(thisBackupId.substring(thisBackupId.lastIndexOf("_") + 1));
- Long otherTS = new Long(otherBackupId.substring(otherBackupId.lastIndexOf("_") + 1));
- return thisTS.compareTo(otherTS);
- }
- }
-
- // manifest version
- private String version = MANIFEST_VERSION;
-
- // hadoop hbase configuration
- protected Configuration config = null;
-
- // backup root directory
- private String rootDir = null;
-
- // backup image directory
- private String tableBackupDir = null;
-
- // backup log directory if this is an incremental backup
- private String logBackupDir = null;
-
- // backup token
- private String token;
-
- // backup type, full or incremental
- private String type;
-
- // the table set for the backup
- private ArrayList<String> tableSet;
-
- // actual start timestamp of the backup process
- private long startTs;
-
- // actual complete timestamp of the backup process
- private long completeTs;
-
- // total bytes for table backup image
- private long tableBytes;
-
- // total bytes for the backed-up logs for incremental backup
- private long logBytes;
-
- // the region server timestamp for tables:
- // <table, <rs, timestamp>>
- private Map<String, HashMap<String, String>> incrTimeRanges;
-
- // dependency of this backup, including all the dependent images to do PIT recovery
- private Map<String, BackupImage> dependency;
-
- // the state of backup image
- private String imageState;
-
- // the indicator of the image compaction
- private boolean isCompacted = false;
-
- // the merge chain of the original backups, null if not a merged backup
- private LinkedList<String> mergeChain;
-
- /**
- * Construct manifest for a ongoing backup.
- * @param backupCtx The ongoing backup context
- */
- public BackupManifest(BackupContext backupCtx) {
- this.token = backupCtx.getBackupId();
- this.type = backupCtx.getType();
- this.rootDir = backupCtx.getTargetRootDir();
- if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- this.logBackupDir = backupCtx.getHLogTargetDir();
- this.logBytes = backupCtx.getTotalBytesCopied();
- }
- this.startTs = backupCtx.getStartTs();
- this.completeTs = backupCtx.getEndTs();
- this.loadTableSet(backupCtx.getTableListAsString());
- this.setImageOriginal();
- }
-
- /**
- * Construct a table level manifest for a backup of the named table.
- * @param backupCtx The ongoing backup context
- */
- public BackupManifest(BackupContext backupCtx, String table) {
- this.token = backupCtx.getBackupId();
- this.type = backupCtx.getType();
- this.rootDir = backupCtx.getTargetRootDir();
- this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir();
- if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- this.logBackupDir = backupCtx.getHLogTargetDir();
- this.logBytes = backupCtx.getTotalBytesCopied();
- }
- this.startTs = backupCtx.getStartTs();
- this.completeTs = backupCtx.getEndTs();
- this.loadTableSet(table);
- this.setImageOriginal();
- }
-
- /**
- * Construct manifest from a backup directory.
- * @param conf configuration
- * @param backupPath backup path
- * @throws BackupException exception
- */
- public BackupManifest(Configuration conf, Path backupPath) throws BackupException {
-
- LOG.debug("Loading manifest from: " + backupPath.toString());
- // The input backupDir may not exactly be the backup table dir.
- // It could be the backup log dir where there is also a manifest file stored.
- // This variable's purpose is to keep the correct and original location so
- // that we can store/persist it.
- this.tableBackupDir = backupPath.toString();
- this.config = conf;
- try {
-
- FileSystem fs = backupPath.getFileSystem(conf);
- FileStatus[] subFiles = FSUtils.listStatus(fs, backupPath);
- if (subFiles == null) {
- String errorMsg = backupPath.toString() + " does not exist";
- LOG.error(errorMsg);
- throw new IOException(errorMsg);
- }
- for (FileStatus subFile : subFiles) {
- if (subFile.getPath().getName().equals(FILE_NAME)) {
-
- // load and set manifest field from file content
- FSDataInputStream in = fs.open(subFile.getPath());
- Properties props = new Properties();
- try {
- props.load(in);
- } catch (IOException e) {
- LOG.error("Error when loading from manifest file!");
- throw e;
- } finally {
- in.close();
- }
-
- this.version = props.getProperty(TAG_VERSION);
- this.token = props.getProperty(TAG_BACKUPID);
- this.type = props.getProperty(TAG_BACKUPTYPE);
- // Here the parameter backupDir is where the manifest file is.
- // There should always be a manifest file under:
- // backupRootDir/namespace/table/backupId/.backup.manifest
- this.rootDir = backupPath.getParent().getParent().getParent().toString();
-
- Path p = backupPath.getParent();
- if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
- this.rootDir = p.getParent().toString();
- } else {
- this.rootDir = p.getParent().getParent().toString();
- }
-
- this.loadTableSet(props.getProperty(TAG_TABLESET));
-
- this.startTs = Long.parseLong(props.getProperty(TAG_STARTTS));
- this.completeTs = Long.parseLong(props.getProperty(TAG_COMPLETETS));
- this.tableBytes = Long.parseLong(props.getProperty(TAG_TABLEBYTES));
- if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- this.logBytes = (Long.parseLong(props.getProperty(TAG_LOGBYTES)));
- LOG.debug("convert will be implemented by future jira");
- }
- this.loadIncrementalTimeRanges(props.getProperty(TAG_INCRTIMERANGE));
- this.loadDependency(props.getProperty(TAG_DEPENDENCY));
- this.imageState = props.getProperty(TAG_IMAGESTATE);
- this.isCompacted =
- props.getProperty(TAG_COMPACTION).equalsIgnoreCase("TRUE") ? true : false;
- LOG.debug("merge and from existing snapshot will be implemented by future jira");
- LOG.debug("Loaded manifest instance from manifest file: "
- + FSUtils.getPath(subFile.getPath()));
- return;
- }
- }
- String errorMsg = "No manifest file found in: " + backupPath.toString();
- LOG.error(errorMsg);
- throw new IOException(errorMsg);
-
- } catch (IOException e) {
- throw new BackupException(e.getMessage());
- }
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- * Load table set from a table set list string (t1;t2;t3;...).
- * @param tableSetStr Table set list string
- */
- private void loadTableSet(String tableSetStr) {
-
- LOG.debug("Loading table set: " + tableSetStr);
-
- String[] tableSet = tableSetStr.split(";");
- this.tableSet = this.getTableSet();
- if (this.tableSet.size() > 0) {
- this.tableSet.clear();
- }
- for (int i = 0; i < tableSet.length; i++) {
- this.tableSet.add(tableSet[i]);
- }
-
- LOG.debug(tableSet.length + " tables exist in table set.");
- }
-
- public void setImageOriginal() {
- this.imageState = IMAGE_STATE_ORIG;
- }
-
- /**
- * Get the table set of this image.
- * @return The table set list
- */
- public ArrayList<String> getTableSet() {
- if (this.tableSet == null) {
- this.tableSet = new ArrayList<String>();
- }
- return this.tableSet;
- }
-
- /**
- * Persist the manifest file.
- * @throws IOException IOException when storing the manifest file.
- */
- public void store(Configuration conf) throws BackupException {
- Properties props = new Properties();
- props.setProperty(TAG_VERSION, this.version);
- props.setProperty(TAG_BACKUPID, this.token);
- props.setProperty(TAG_BACKUPTYPE, this.type);
- props.setProperty(TAG_TABLESET, this.getTableSetStr());
- LOG.debug("convert will be supported in future jira");
- // String convertedTables = this.getConvertedTableSetStr();
- // if (convertedTables != null )
- // props.setProperty(TAG_CONVERTEDTABLESET, convertedTables);
- props.setProperty(TAG_STARTTS, Long.toString(this.startTs));
- props.setProperty(TAG_COMPLETETS, Long.toString(this.completeTs));
- props.setProperty(TAG_TABLEBYTES, Long.toString(this.tableBytes));
- if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- props.setProperty(TAG_LOGBYTES, Long.toString(this.logBytes));
- }
- props.setProperty(TAG_INCRTIMERANGE, this.getIncrTimestampStr());
- props.setProperty(TAG_DEPENDENCY, this.getDependencyStr());
- props.setProperty(TAG_IMAGESTATE, this.getImageState());
- props.setProperty(TAG_COMPACTION, this.isCompacted ? "TRUE" : "FALSE");
- LOG.debug("merge will be supported in future jira");
- // props.setProperty(TAG_MERGECHAIN, this.getMergeChainStr());
- LOG.debug("backup from existing snapshot will be supported in future jira");
- // props.setProperty(TAG_FROMSNAPSHOT, this.isFromSnapshot() ? "TRUE" : "FALSE");
-
- // write the file, overwrite if already exist
- Path manifestFilePath =
- new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir)
- + File.separator + FILE_NAME);
- try {
- FSDataOutputStream out = manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);
- props.store(out, "HBase backup manifest.");
- out.close();
- } catch (IOException e) {
- throw new BackupException(e.getMessage());
- }
-
- LOG.debug("Manifest file stored to " + this.tableBackupDir != null ? this.tableBackupDir
- : this.logBackupDir + File.separator + FILE_NAME);
- }
-
- /**
- * Get the table set string in the format of t1;t2;t3...
- */
- private String getTableSetStr() {
- return BackupUtil.concat(getTableSet(), ";");
- }
-
- public String getImageState() {
- return imageState;
- }
-
- public String getVersion() {
- return version;
- }
-
- /**
- * Get this backup image.
- * @return the backup image.
- */
- public BackupImage getBackupImage() {
- return this.getDependency().get(this.token);
- }
-
- /**
- * Add dependent backup image for this backup.
- * @param image The direct dependent backup image
- */
- public void addDependentImage(BackupImage image) {
- this.getDependency().get(this.token).addAncestor(image);
- this.setDependencyMap(this.getDependency(), image);
- }
-
- /**
- * Get the dependency' string in the json format.
- */
- private String getDependencyStr() {
- BackupImage thisImage = this.getDependency().get(this.token);
- if (thisImage == null) {
- LOG.warn("There is no dependency set yet.");
- return null;
- }
-
- ObjectMapper mapper = new ObjectMapper();
- try {
- return mapper.writeValueAsString(thisImage);
- } catch (JsonGenerationException e) {
- LOG.error("Error when generating dependency string from backup image.", e);
- return ERROR_DEPENDENCY;
- } catch (JsonMappingException e) {
- LOG.error("Error when generating dependency string from backup image.", e);
- return ERROR_DEPENDENCY;
- } catch (IOException e) {
- LOG.error("Error when generating dependency string from backup image.", e);
- return ERROR_DEPENDENCY;
- }
- }
-
- /**
- * Get all dependent backup images. The image of this backup is also contained.
- * @return The dependent backup images map
- */
- public Map<String, BackupImage> getDependency() {
- if (this.dependency == null) {
- this.dependency = new HashMap<String, BackupImage>();
- LOG.debug(this.rootDir + " " + this.token + " " + this.type);
- this.dependency.put(this.token,
- new BackupImage(this.token, this.type, this.rootDir, this.getTableSetStr(), this.startTs,
- this.completeTs));
- }
- return this.dependency;
- }
-
- /**
- * Set the incremental timestamp map directly.
- * @param incrTimestampMap timestamp map
- */
- public void setIncrTimestampMap(HashMap<String, HashMap<String, String>> incrTimestampMap) {
- this.incrTimeRanges = incrTimestampMap;
- }
-
- /**
- * Get the incremental time range string in the format of:
- * t1,rs1:ts,rs2:ts,...;t2,rs1:ts,rs2:ts,...;t3,rs1:ts,rs2:ts,...
- */
- private String getIncrTimestampStr() {
- StringBuilder sb = new StringBuilder();
- for (Entry<String, HashMap<String, String>> tableEntry : this.getIncrTimestamps().entrySet()) {
- sb.append(tableEntry.getKey() + ","); // table
- for (Entry<String, String> rsEntry : tableEntry.getValue().entrySet()) {
- sb.append(rsEntry.getKey() + ":"); // region server
- sb.append(rsEntry.getValue() + ","); // timestamp
- }
- if (sb.length() > 1 && sb.charAt(sb.length() - 1) == ',') {
- sb.deleteCharAt(sb.length() - 1);
- }
- sb.append(";");
- }
- if (sb.length() > 1 && sb.charAt(sb.length() - 1) == ';') {
- sb.deleteCharAt(sb.length() - 1);
- }
- return sb.toString();
- }
-
- public Map<String, HashMap<String, String>> getIncrTimestamps() {
- if (this.incrTimeRanges == null) {
- this.incrTimeRanges = new HashMap<String, HashMap<String, String>>();
- }
- return this.incrTimeRanges;
- }
-
- /**
- * Load incremental timestamps from a given string, and store them in the collection. The
- * timestamps in string is in the format of
- * t1,rs1:ts,rs2:ts,...;t2,rs1:ts,rs2:ts,...;t3,rs1:ts,rs2:ts,...
- * @param timeRangesInStr Incremental time ranges in string
- */
- private void loadIncrementalTimeRanges(String timeRangesStr) throws IOException {
-
- LOG.debug("Loading table's incremental time ranges of region servers from string in manifest: "
- + timeRangesStr);
-
- Map<String, HashMap<String, String>> timeRangeMap = this.getIncrTimestamps();
-
- String[] entriesOfTables = timeRangesStr.split(";");
- for (int i = 0; i < entriesOfTables.length; i++) {
- String[] itemsForTable = entriesOfTables[i].split(",");
-
- // validate the incremental timestamps string format for a table:
- // t1,rs1:ts,rs2:ts,...
- if (itemsForTable.length < 1) {
- String errorMsg = "Wrong incremental time range format: " + timeRangesStr;
- LOG.error(errorMsg);
- throw new IOException(errorMsg);
- }
-
- HashMap<String, String> rsTimestampMap = new HashMap<String, String>();
- for (int j = 1; j < itemsForTable.length; j++) {
- String[] rsTsEntry = itemsForTable[j].split(":");
-
- // validate the incremental timestamps string format for a region server:
- // rs1:ts
- if (rsTsEntry.length != 2) {
- String errorMsg = "Wrong incremental timestamp format: " + itemsForTable[j];
- LOG.error(errorMsg);
- throw new IOException(errorMsg);
- }
-
- // an entry for timestamp of a region server
- rsTimestampMap.put(rsTsEntry[0], rsTsEntry[1]);
- }
-
- timeRangeMap.put(itemsForTable[0], rsTimestampMap);
- }
-
- // all entries have been loaded
- LOG.debug(entriesOfTables.length + " tables' incremental time ranges have been loaded.");
- }
-
- /**
- * Get the image list of this backup for restore in time order.
- * @param reverse If true, then output in reverse order, otherwise in time order from old to new
- * @return the backup image list for restore in time order
- */
- public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
- TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>();
- for (BackupImage image : this.getDependency().values()) {
- restoreImages.put(Long.valueOf(image.startTs), image);
- }
- return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values())
- : (restoreImages.values()));
- }
-
- /**
- * Get the dependent image list for a specific table of this backup in time order from old to new
- * if want to restore to this backup image level.
- * @param table table
- * @return the backup image list for a table in time order
- */
- public ArrayList<BackupImage> getDependentListByTable(String table) {
- ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
- ArrayList<BackupImage> imageList = getRestoreDependentList(true);
- for (BackupImage image : imageList) {
- if (image.hasTable(table)) {
- tableImageList.add(image);
- if (image.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
- break;
- }
- }
- }
- Collections.reverse(tableImageList);
- return tableImageList;
- }
-
- /**
- * Get the full dependent image list in the whole dependency scope for a specific table of this
- * backup in time order from old to new.
- * @param table table
- * @return the full backup image list for a table in time order in the whole scope of the
- * dependency of this image
- */
- public ArrayList<BackupImage> getAllDependentListByTable(String table) {
- ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
- ArrayList<BackupImage> imageList = getRestoreDependentList(false);
- for (BackupImage image : imageList) {
- if (image.hasTable(table)) {
- tableImageList.add(image);
- }
- }
- return tableImageList;
- }
-
- /**
- * Load dependency from a dependency json string.
- * @param dependencyStr The dependency string
- * @throws IOException exception
- */
- private void loadDependency(String dependencyStr) throws IOException {
-
- LOG.debug("Loading dependency: " + dependencyStr);
-
- String msg = "Dependency is broken in the manifest.";
- if (dependencyStr.equals(ERROR_DEPENDENCY)) {
- throw new IOException(msg);
- }
-
- ObjectMapper mapper = new ObjectMapper();
- BackupImage image = null;
- try {
- image = mapper.readValue(dependencyStr, BackupImage.class);
- } catch (JsonParseException e) {
- LOG.error(msg);
- throw new IOException(e.getMessage());
- } catch (JsonMappingException e) {
- LOG.error(msg);
- throw new IOException(e.getMessage());
- } catch (IOException e) {
- LOG.error(msg);
- throw new IOException(e.getMessage());
- }
- LOG.debug("Manifest's current backup image information:");
- LOG.debug(" Token: " + image.getBackupId());
- LOG.debug(" Backup directory: " + image.getRootDir());
- this.setDependencyMap(this.getDependency(), image);
-
- LOG.debug("Dependent images map:");
- for (Entry<String, BackupImage> entry : this.getDependency().entrySet()) {
- LOG.debug(" " + entry.getKey() + " : " + entry.getValue().getBackupId() + " -- "
- + entry.getValue().getRootDir());
- }
-
- LOG.debug("Dependency has been loaded.");
- }
-
- /**
- * Recursively set the dependency map of the backup images.
- * @param map The dependency map
- * @param image The backup image
- */
- private void setDependencyMap(Map<String, BackupImage> map, BackupImage image) {
- if (image == null) {
- return;
- } else {
- map.put(image.getBackupId(), image);
- for (BackupImage img : image.getAncestors()) {
- setDependencyMap(map, img);
- }
- }
- }
-
- /**
- * Check whether backup image1 could cover backup image2 or not.
- * @param image1 backup image 1
- * @param image2 backup image 2
- * @return true if image1 can cover image2, otherwise false
- */
- public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
- // image1 can cover image2 only when the following conditions are satisfied:
- // - image1 must not be an incremental image;
- // - image1 must be taken after image2 has been taken;
- // - table set of image1 must cover the table set of image2.
- if (image1.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- return false;
- }
- if (image1.getStartTs() < image2.getStartTs()) {
- return false;
- }
- String[] image1TableSet = image1.getTableSet().split(";");
- String[] image2TableSet = image2.getTableSet().split(";");
- boolean found = false;
- for (int i = 0; i < image2TableSet.length; i++) {
- found = false;
- for (int j = 0; j < image1TableSet.length; j++) {
- if (image2TableSet[i].equals(image1TableSet[j])) {
- found = true;
- break;
- }
- }
- if (!found) {
- return false;
- }
- }
-
- LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
- return true;
- }
-
- /**
- * Check whether backup image set could cover a backup image or not.
- * @param fullImages The backup image set
- * @param image The target backup image
- * @return true if fullImages can cover image, otherwise false
- */
- public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
- // fullImages can cover image only when the following conditions are satisfied:
- // - each image of fullImages must not be an incremental image;
- // - each image of fullImages must be taken after image has been taken;
- // - sum table set of fullImages must cover the table set of image.
- for (BackupImage image1 : fullImages) {
- if (image1.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- return false;
- }
- if (image1.getStartTs() < image.getStartTs()) {
- return false;
- }
- }
-
- ArrayList<String> image1TableSet = new ArrayList<String>();
- for (BackupImage image1 : fullImages) {
- String[] tableSet = image1.getTableSet().split(";");
- for (String table : tableSet) {
- image1TableSet.add(table);
- }
- }
- ArrayList<String> image2TableSet = new ArrayList<String>();
- String[] tableSet = image.getTableSet().split(";");
- for (String table : tableSet) {
- image2TableSet.add(table);
- }
-
- for (int i = 0; i < image2TableSet.size(); i++) {
- if (image1TableSet.contains(image2TableSet.get(i)) == false) {
- return false;
- }
- }
-
- LOG.debug("Full image set can cover image " + image.getBackupId());
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
deleted file mode 100644
index 9610b27..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * HConstants holds a bunch of HBase Backup and Restore constants
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public final class BackupRestoreConstants {
-
- // constants for znode data keys in backup znode
- public static final String BACKUP_PROGRESS = "progress";
- public static final String BACKUP_START_TIME = "startTs";
- public static final String BACKUP_INPROGRESS_PHASE = "phase";
- public static final String BACKUP_COMPLETE_TIME = "completeTs";
- public static final String BACKUP_FAIL_TIME = "failedTs";
- public static final String BACKUP_FAIL_PHASE = "failedphase";
- public static final String BACKUP_FAIL_MSG = "failedmessage";
- public static final String BACKUP_ROOT_PATH = "targetRootDir";
- public static final String BACKUP_REQUEST_TABLE_LIST = "tablelist";
- public static final String BACKUP_REQUEST_TYPE = "type";
- public static final String BACKUP_BYTES_COPIED = "bytescopied";
- public static final String BACKUP_ANCESTORS = "ancestors";
- public static final String BACKUP_EXISTINGSNAPSHOT = "snapshot";
-
- public static final String BACKUP_TYPE_FULL = "full";
- public static final String BACKUP_TYPE_INCR = "incremental";
-
- // delimiter in tablename list in restore command
- public static final String TABLENAME_DELIMITER_IN_COMMAND = ",";
-
- // delimiter in znode data
- public static final String ZNODE_DATA_DELIMITER = ",";
-
- public static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
-
- public static final String BACKUPID_PREFIX = "backup_";
-
- public static enum BACKUP_COMMAND {
- CREATE, CANCEL, DELETE, DESCRIBE, HISTORY, STATUS, CONVERT, MERGE, STOP, SHOW, HELP,
- }
-
- private BackupRestoreConstants() {
- // Can't be instantiated with this ctor.
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java
deleted file mode 100644
index 1c38cf4..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyService;
-import org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreService;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.util.ReflectionUtils;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class BackupRestoreServiceFactory {
-
- public final static String HBASE_INCR_RESTORE_IMPL_CLASS = "hbase.incremental.restore.class";
- public final static String HBASE_BACKUP_COPY_IMPL_CLASS = "hbase.backup.copy.class";
-
- private BackupRestoreServiceFactory(){
- throw new AssertionError("Instantiating utility class...");
- }
-
- /**
- * Gets incremental restore service
- * @param conf - configuration
- * @return incremental backup service instance
- */
- public static IncrementalRestoreService getIncrementalRestoreService(Configuration conf) {
- Class<? extends IncrementalRestoreService> cls =
- conf.getClass(HBASE_INCR_RESTORE_IMPL_CLASS, MapReduceRestoreService.class,
- IncrementalRestoreService.class);
- return ReflectionUtils.newInstance(cls, conf);
- }
-
- /**
- * Gets backup copy service
- * @param conf - configuration
- * @return backup copy service
- */
- public static BackupCopyService getBackupCopyService(Configuration conf) {
- Class<? extends BackupCopyService> cls =
- conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyService.class,
- BackupCopyService.class);
- return ReflectionUtils.newInstance(cls, conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
deleted file mode 100644
index c26e1d4..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.Serializable;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * Backup status and related information encapsulated for a table.
- * At this moment only TargetDir and SnapshotName is encapsulated here.
- * future Jira will be implemented for progress, bytesCopies, phase, etc.
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupStatus implements Serializable {
-
- private static final long serialVersionUID = -5968397963548535982L;
-
- // table name for backup
- private String table;
-
- // target directory of the backup image for this table
- private String targetDir;
-
- // snapshot name for offline/online snapshot
- private String snapshotName = null;
-
- public BackupStatus(String table, String targetRootDir, String backupId) {
- this.table = table;
- this.targetDir = HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
- }
-
- public String getSnapshotName() {
- return snapshotName;
- }
-
- public void setSnapshotName(String snapshotName) {
- this.snapshotName = snapshotName;
- }
-
- public String getTargetDir() {
- return targetDir;
- }
-
- public String getTable() {
- return table;
- }
-}
[12/29] hbase git commit: HBASE-14030 Revert due to pending review
comments
Posted by sy...@apache.org.
HBASE-14030 Revert due to pending review comments
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/449fb812
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/449fb812
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/449fb812
Branch: refs/heads/hbase-12439
Commit: 449fb81286f405cd7b373499c5db65ea61f9ffab
Parents: de69f0d
Author: tedyu <yu...@gmail.com>
Authored: Mon Dec 28 01:08:02 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Mon Dec 28 01:08:02 2015 -0800
----------------------------------------------------------------------
bin/hbase | 6 -
.../org/apache/hadoop/hbase/HConstants.java | 445 +++++-----
hbase-server/pom.xml | 10 -
.../hadoop/hbase/backup/BackupClient.java | 338 --------
.../hadoop/hbase/backup/BackupCommands.java | 147 ----
.../hadoop/hbase/backup/BackupContext.java | 318 --------
.../hadoop/hbase/backup/BackupCopyService.java | 37 -
.../hadoop/hbase/backup/BackupException.java | 85 --
.../hadoop/hbase/backup/BackupHandler.java | 744 -----------------
.../hadoop/hbase/backup/BackupManager.java | 488 -----------
.../hadoop/hbase/backup/BackupManifest.java | 814 -------------------
.../hbase/backup/BackupRestoreConstants.java | 66 --
.../backup/BackupRestoreServiceFactory.java | 61 --
.../hadoop/hbase/backup/BackupStatus.java | 67 --
.../hadoop/hbase/backup/BackupSystemTable.java | 642 ---------------
.../hbase/backup/BackupSystemTableHelper.java | 314 -------
.../apache/hadoop/hbase/backup/BackupUtil.java | 564 -------------
.../hadoop/hbase/backup/HBackupFileSystem.java | 511 ------------
.../hbase/backup/IncrementalBackupManager.java | 269 ------
.../hbase/backup/IncrementalRestoreService.java | 33 -
.../hadoop/hbase/backup/RestoreClient.java | 496 -----------
.../apache/hadoop/hbase/backup/RestoreUtil.java | 503 ------------
.../mapreduce/MapReduceBackupCopyService.java | 292 -------
.../mapreduce/MapReduceRestoreService.java | 72 --
.../hbase/backup/master/BackupLogCleaner.java | 121 ---
.../master/LogRollMasterProcedureManager.java | 129 ---
.../regionserver/LogRollBackupSubprocedure.java | 138 ----
.../LogRollBackupSubprocedurePool.java | 137 ----
.../LogRollRegionServerProcedureManager.java | 168 ----
.../BaseCoordinatedStateManager.java | 17 -
.../coordination/ZkCoordinatedStateManager.java | 20 +-
.../hadoop/hbase/mapreduce/WALPlayer.java | 54 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 2 -
.../procedure/RegionServerProcedureManager.java | 2 +-
.../RegionServerProcedureManagerHost.java | 3 +-
.../procedure/ZKProcedureCoordinatorRpcs.java | 2 +-
.../hbase/procedure/ZKProcedureMemberRpcs.java | 83 +-
.../RegionServerFlushTableProcedureManager.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 4 +-
.../snapshot/RegionServerSnapshotManager.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 67 +-
.../hadoop/hbase/snapshot/SnapshotCopy.java | 42 -
.../hadoop/hbase/wal/DefaultWALProvider.java | 7 +-
.../hadoop/hbase/backup/TestBackupBase.java | 194 -----
.../hbase/backup/TestBackupBoundaryTests.java | 99 ---
.../hbase/backup/TestBackupLogCleaner.java | 161 ----
.../hbase/backup/TestBackupSystemTable.java | 341 --------
.../hadoop/hbase/backup/TestFullBackup.java | 83 --
.../hadoop/hbase/backup/TestFullRestore.java | 166 ----
.../hbase/backup/TestIncrementalBackup.java | 179 ----
.../hadoop/hbase/backup/TestRemoteBackup.java | 44 -
.../hadoop/hbase/backup/TestRemoteRestore.java | 54 --
.../hbase/backup/TestRestoreBoundaryTests.java | 85 --
.../procedure/SimpleRSProcedureManager.java | 2 +-
54 files changed, 341 insertions(+), 9389 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 9fd903a..5064451 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -99,8 +99,6 @@ if [ $# = 0 ]; then
echo " pe Run PerformanceEvaluation"
echo " ltt Run LoadTestTool"
echo " version Print the version"
- echo " backup backup tables for recovery"
- echo " restore restore tables from existing backup image"
echo " CLASSNAME Run the class named CLASSNAME"
exit 1
fi
@@ -305,10 +303,6 @@ elif [ "$COMMAND" = "hfile" ] ; then
CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter'
elif [ "$COMMAND" = "zkcli" ] ; then
CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer"
-elif [ "$COMMAND" = "backup" ] ; then
- CLASS='org.apache.hadoop.hbase.backup.BackupClient'
-elif [ "$COMMAND" = "restore" ] ; then
- CLASS='org.apache.hadoop.hbase.backup.RestoreClient'
elif [ "$COMMAND" = "upgrade" ] ; then
echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0."
echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading."
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 0f572e6..6fafad3 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -40,11 +40,11 @@ import org.apache.hadoop.hbase.util.Bytes;
public final class HConstants {
// NOTICE!!!! Please do not add a constants here, unless they are referenced by a lot of classes.
- // Bytes.UTF8_ENCODING should be updated if this changed
+ //Bytes.UTF8_ENCODING should be updated if this changed
/** When we encode strings, we always specify UTF8 encoding */
public static final String UTF8_ENCODING = "UTF-8";
- // Bytes.UTF8_CHARSET should be updated if this changed
+ //Bytes.UTF8_CHARSET should be updated if this changed
/** When we encode strings, we always specify UTF8 encoding */
public static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
/**
@@ -55,9 +55,9 @@ public final class HConstants {
/** Used as a magic return value while optimized index key feature enabled(HBASE-7845) */
public final static int INDEX_KEY_MAGIC = -2;
/*
- * Name of directory that holds recovered edits written by the wal log splitting code, one per
- * region
- */
+ * Name of directory that holds recovered edits written by the wal log
+ * splitting code, one per region
+ */
public static final String RECOVERED_EDITS_DIR = "recovered.edits";
/**
* The first four bytes of Hadoop RPC connections
@@ -70,24 +70,27 @@ public final class HConstants {
/** The size data structures with minor version is 0 */
public static final int HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
+ Bytes.SIZEOF_LONG;
- /**
- * The size of a version 2 HFile block header, minor version 1. There is a 1 byte checksum type,
- * followed by a 4 byte bytesPerChecksum followed by another 4 byte value to store
- * sizeofDataOnDisk.
+ /** The size of a version 2 HFile block header, minor version 1.
+ * There is a 1 byte checksum type, followed by a 4 byte bytesPerChecksum
+ * followed by another 4 byte value to store sizeofDataOnDisk.
*/
- public static final int HFILEBLOCK_HEADER_SIZE = HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM
- + Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT;
+ public static final int HFILEBLOCK_HEADER_SIZE = HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM +
+ Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT;
/** Just an array of bytes of the right size. */
public static final byte[] HFILEBLOCK_DUMMY_HEADER = new byte[HFILEBLOCK_HEADER_SIZE];
- // End HFileBlockConstants.
+ //End HFileBlockConstants.
/**
* Status codes used for return values of bulk operations.
*/
@InterfaceAudience.Private
public enum OperationStatusCode {
- NOT_RUN, SUCCESS, BAD_FAMILY, SANITY_CHECK_FAILURE, FAILURE;
+ NOT_RUN,
+ SUCCESS,
+ BAD_FAMILY,
+ SANITY_CHECK_FAILURE,
+ FAILURE;
}
/** long constant for zero */
@@ -101,16 +104,19 @@ public final class HConstants {
public static final String VERSION_FILE_NAME = "hbase.version";
/**
- * Current version of file system. Version 4 supports only one kind of bloom filter. Version 5
- * changes versions in catalog table regions. Version 6 enables blockcaching on catalog tables.
- * Version 7 introduces hfile -- hbase 0.19 to 0.20.. Version 8 introduces namespace
+ * Current version of file system.
+ * Version 4 supports only one kind of bloom filter.
+ * Version 5 changes versions in catalog table regions.
+ * Version 6 enables blockcaching on catalog tables.
+ * Version 7 introduces hfile -- hbase 0.19 to 0.20..
+ * Version 8 introduces namespace
*/
// public static final String FILE_SYSTEM_VERSION = "6";
public static final String FILE_SYSTEM_VERSION = "8";
// Configuration parameters
- // TODO: Is having HBase homed on port 60k OK?
+ //TODO: Is having HBase homed on port 60k OK?
/** Cluster is in distributed mode or not */
public static final String CLUSTER_DISTRIBUTED = "hbase.cluster.distributed";
@@ -125,10 +131,12 @@ public final class HConstants {
public static final String ENSEMBLE_TABLE_NAME = "hbase:ensemble";
/** Config for pluggable region normalizer */
- public static final String HBASE_MASTER_NORMALIZER_CLASS = "hbase.master.normalizer.class";
+ public static final String HBASE_MASTER_NORMALIZER_CLASS =
+ "hbase.master.normalizer.class";
/** Config for enabling/disabling pluggable region normalizer */
- public static final String HBASE_NORMALIZER_ENABLED = "hbase.normalizer.enabled";
+ public static final String HBASE_NORMALIZER_ENABLED =
+ "hbase.normalizer.enabled";
/** Cluster is standalone or pseudo-distributed */
public static final boolean CLUSTER_IS_LOCAL = false;
@@ -166,18 +174,21 @@ public final class HConstants {
public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
/** Common prefix of ZooKeeper configuration properties */
- public static final String ZK_CFG_PROPERTY_PREFIX = "hbase.zookeeper.property.";
+ public static final String ZK_CFG_PROPERTY_PREFIX =
+ "hbase.zookeeper.property.";
- public static final int ZK_CFG_PROPERTY_PREFIX_LEN = ZK_CFG_PROPERTY_PREFIX.length();
+ public static final int ZK_CFG_PROPERTY_PREFIX_LEN =
+ ZK_CFG_PROPERTY_PREFIX.length();
/**
- * The ZK client port key in the ZK properties map. The name reflects the fact that this is not an
- * HBase configuration key.
+ * The ZK client port key in the ZK properties map. The name reflects the
+ * fact that this is not an HBase configuration key.
*/
public static final String CLIENT_PORT_STR = "clientPort";
/** Parameter name for the client port that the zookeeper listens on */
- public static final String ZOOKEEPER_CLIENT_PORT = ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR;
+ public static final String ZOOKEEPER_CLIENT_PORT =
+ ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR;
/** Default client port that the zookeeper listens on */
public static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 2181;
@@ -197,15 +208,19 @@ public final class HConstants {
public static final String DEFAULT_ZOOKEEPER_ZNODE_PARENT = "/hbase";
/**
- * Parameter name for the limit on concurrent client-side zookeeper connections
+ * Parameter name for the limit on concurrent client-side zookeeper
+ * connections
*/
- public static final String ZOOKEEPER_MAX_CLIENT_CNXNS = ZK_CFG_PROPERTY_PREFIX + "maxClientCnxns";
+ public static final String ZOOKEEPER_MAX_CLIENT_CNXNS =
+ ZK_CFG_PROPERTY_PREFIX + "maxClientCnxns";
/** Parameter name for the ZK data directory */
- public static final String ZOOKEEPER_DATA_DIR = ZK_CFG_PROPERTY_PREFIX + "dataDir";
+ public static final String ZOOKEEPER_DATA_DIR =
+ ZK_CFG_PROPERTY_PREFIX + "dataDir";
/** Parameter name for the ZK tick time */
- public static final String ZOOKEEPER_TICK_TIME = ZK_CFG_PROPERTY_PREFIX + "tickTime";
+ public static final String ZOOKEEPER_TICK_TIME =
+ ZK_CFG_PROPERTY_PREFIX + "tickTime";
/** Default limit on concurrent client-side zookeeper connections */
public static final int DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS = 300;
@@ -229,19 +244,21 @@ public final class HConstants {
public static final int DEFAULT_REGIONSERVER_INFOPORT = 16030;
/** A configuration key for regionserver info port */
- public static final String REGIONSERVER_INFO_PORT = "hbase.regionserver.info.port";
+ public static final String REGIONSERVER_INFO_PORT =
+ "hbase.regionserver.info.port";
/** A flag that enables automatic selection of regionserver info port */
- public static final String REGIONSERVER_INFO_PORT_AUTO = REGIONSERVER_INFO_PORT + ".auto";
+ public static final String REGIONSERVER_INFO_PORT_AUTO =
+ REGIONSERVER_INFO_PORT + ".auto";
/** Parameter name for what region server implementation to use. */
- public static final String REGION_SERVER_IMPL = "hbase.regionserver.impl";
+ public static final String REGION_SERVER_IMPL= "hbase.regionserver.impl";
/** Parameter name for what master implementation to use. */
- public static final String MASTER_IMPL = "hbase.master.impl";
+ public static final String MASTER_IMPL= "hbase.master.impl";
/** Parameter name for what hbase client implementation to use. */
- public static final String HBASECLIENT_IMPL = "hbase.hbaseclient.impl";
+ public static final String HBASECLIENT_IMPL= "hbase.hbaseclient.impl";
/** Parameter name for how often threads should wake up */
public static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency";
@@ -276,7 +293,7 @@ public final class HConstants {
/** Parameter name for HBase client operation timeout, which overrides RPC timeout */
public static final String HBASE_CLIENT_META_OPERATION_TIMEOUT =
- "hbase.client.meta.operation.timeout";
+ "hbase.client.meta.operation.timeout";
/** Default HBase client operation timeout, which is tantamount to a blocking call */
public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = 1200000;
@@ -299,8 +316,9 @@ public final class HConstants {
public static final String MIGRATION_NAME = ".migration";
/**
- * The directory from which co-processor/custom filter jars can be loaded dynamically by the
- * region servers. This value can be overridden by the hbase.dynamic.jars.dir config.
+ * The directory from which co-processor/custom filter jars can be loaded
+ * dynamically by the region servers. This value can be overridden by the
+ * hbase.dynamic.jars.dir config.
*/
public static final String LIB_DIR = "lib";
@@ -308,7 +326,8 @@ public final class HConstants {
public static final String HREGION_COMPACTIONDIR_NAME = "compaction.dir";
/** Conf key for the max file size after which we split the region */
- public static final String HREGION_MAX_FILESIZE = "hbase.hregion.max.filesize";
+ public static final String HREGION_MAX_FILESIZE =
+ "hbase.hregion.max.filesize";
/** Default maximum file size */
public static final long DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024L;
@@ -324,24 +343,25 @@ public final class HConstants {
public static final long TABLE_MAX_ROWSIZE_DEFAULT = 1024 * 1024 * 1024L;
/**
- * The max number of threads used for opening and closing stores or store files in parallel
+ * The max number of threads used for opening and closing stores or store
+ * files in parallel
*/
public static final String HSTORE_OPEN_AND_CLOSE_THREADS_MAX =
- "hbase.hstore.open.and.close.threads.max";
+ "hbase.hstore.open.and.close.threads.max";
/**
- * The default number for the max number of threads used for opening and closing stores or store
- * files in parallel
+ * The default number for the max number of threads used for opening and
+ * closing stores or store files in parallel
*/
public static final int DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX = 1;
/**
- * Block updates if memstore has hbase.hregion.memstore.block.multiplier times
- * hbase.hregion.memstore.flush.size bytes. Useful preventing runaway memstore during spikes in
- * update traffic.
+ * Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ * times hbase.hregion.memstore.flush.size bytes. Useful preventing
+ * runaway memstore during spikes in update traffic.
*/
public static final String HREGION_MEMSTORE_BLOCK_MULTIPLIER =
- "hbase.hregion.memstore.block.multiplier";
+ "hbase.hregion.memstore.block.multiplier";
/**
* Default value for hbase.hregion.memstore.block.multiplier
@@ -349,12 +369,14 @@ public final class HConstants {
public static final int DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER = 4;
/** Conf key for the memstore size at which we flush the memstore */
- public static final String HREGION_MEMSTORE_FLUSH_SIZE = "hbase.hregion.memstore.flush.size";
+ public static final String HREGION_MEMSTORE_FLUSH_SIZE =
+ "hbase.hregion.memstore.flush.size";
public static final String HREGION_EDITS_REPLAY_SKIP_ERRORS =
"hbase.hregion.edits.replay.skip.errors";
- public static final boolean DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS = false;
+ public static final boolean DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS =
+ false;
/** Maximum value length, enforced on KeyValue construction */
public static final int MAXIMUM_VALUE_LENGTH = Integer.MAX_VALUE - 1;
@@ -390,12 +412,12 @@ public final class HConstants {
// be the first to be reassigned if the server(s) they are being served by
// should go down.
+
/**
* The hbase:meta table's name.
* @deprecated For upgrades of 0.94 to 0.96
*/
- @Deprecated
- // for compat from 0.94 -> 0.96.
+ @Deprecated // for compat from 0.94 -> 0.96.
public static final byte[] META_TABLE_NAME = TableName.META_TABLE_NAME.getName();
public static final String BASE_NAMESPACE_DIR = "data";
@@ -403,52 +425,52 @@ public final class HConstants {
/** delimiter used between portions of a region name */
public static final int META_ROW_DELIMITER = ',';
- /** The catalog family as a string */
+ /** The catalog family as a string*/
public static final String CATALOG_FAMILY_STR = "info";
/** The catalog family */
- public static final byte[] CATALOG_FAMILY = Bytes.toBytes(CATALOG_FAMILY_STR);
+ public static final byte [] CATALOG_FAMILY = Bytes.toBytes(CATALOG_FAMILY_STR);
/** The RegionInfo qualifier as a string */
public static final String REGIONINFO_QUALIFIER_STR = "regioninfo";
/** The regioninfo column qualifier */
- public static final byte[] REGIONINFO_QUALIFIER = Bytes.toBytes(REGIONINFO_QUALIFIER_STR);
+ public static final byte [] REGIONINFO_QUALIFIER = Bytes.toBytes(REGIONINFO_QUALIFIER_STR);
/** The server column qualifier */
public static final String SERVER_QUALIFIER_STR = "server";
/** The server column qualifier */
- public static final byte[] SERVER_QUALIFIER = Bytes.toBytes(SERVER_QUALIFIER_STR);
+ public static final byte [] SERVER_QUALIFIER = Bytes.toBytes(SERVER_QUALIFIER_STR);
/** The startcode column qualifier */
public static final String STARTCODE_QUALIFIER_STR = "serverstartcode";
/** The startcode column qualifier */
- public static final byte[] STARTCODE_QUALIFIER = Bytes.toBytes(STARTCODE_QUALIFIER_STR);
+ public static final byte [] STARTCODE_QUALIFIER = Bytes.toBytes(STARTCODE_QUALIFIER_STR);
/** The open seqnum column qualifier */
public static final String SEQNUM_QUALIFIER_STR = "seqnumDuringOpen";
/** The open seqnum column qualifier */
- public static final byte[] SEQNUM_QUALIFIER = Bytes.toBytes(SEQNUM_QUALIFIER_STR);
+ public static final byte [] SEQNUM_QUALIFIER = Bytes.toBytes(SEQNUM_QUALIFIER_STR);
/** The state column qualifier */
public static final String STATE_QUALIFIER_STR = "state";
- public static final byte[] STATE_QUALIFIER = Bytes.toBytes(STATE_QUALIFIER_STR);
+ public static final byte [] STATE_QUALIFIER = Bytes.toBytes(STATE_QUALIFIER_STR);
/**
- * The serverName column qualifier. Its the server where the region is transitioning on, while
- * column server is the server where the region is opened on. They are the same when the region is
- * in state OPEN.
+ * The serverName column qualifier. Its the server where the region is
+ * transitioning on, while column server is the server where the region is
+ * opened on. They are the same when the region is in state OPEN.
*/
public static final String SERVERNAME_QUALIFIER_STR = "sn";
- public static final byte[] SERVERNAME_QUALIFIER = Bytes.toBytes(SERVERNAME_QUALIFIER_STR);
+ public static final byte [] SERVERNAME_QUALIFIER = Bytes.toBytes(SERVERNAME_QUALIFIER_STR);
/** The lower-half split region column qualifier */
- public static final byte[] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
+ public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
/** The upper-half split region column qualifier */
- public static final byte[] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
+ public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
/** The lower-half merge region column qualifier */
public static final byte[] MERGEA_QUALIFIER = Bytes.toBytes("mergeA");
@@ -456,28 +478,32 @@ public final class HConstants {
/** The upper-half merge region column qualifier */
public static final byte[] MERGEB_QUALIFIER = Bytes.toBytes("mergeB");
- /** The catalog family as a string */
+ /** The catalog family as a string*/
public static final String TABLE_FAMILY_STR = "table";
/** The catalog family */
- public static final byte[] TABLE_FAMILY = Bytes.toBytes(TABLE_FAMILY_STR);
+ public static final byte [] TABLE_FAMILY = Bytes.toBytes(TABLE_FAMILY_STR);
/** The serialized table state qualifier */
public static final byte[] TABLE_STATE_QUALIFIER = Bytes.toBytes("state");
+
/**
- * The meta table version column qualifier. We keep current version of the meta table in this
- * column in <code>-ROOT-</code> table: i.e. in the 'info:v' column.
+ * The meta table version column qualifier.
+ * We keep current version of the meta table in this column in <code>-ROOT-</code>
+ * table: i.e. in the 'info:v' column.
*/
- public static final byte[] META_VERSION_QUALIFIER = Bytes.toBytes("v");
+ public static final byte [] META_VERSION_QUALIFIER = Bytes.toBytes("v");
/**
- * The current version of the meta table. - pre-hbase 0.92. There is no META_VERSION column in the
- * root table in this case. The meta has HTableDescriptor serialized into the HRegionInfo; -
- * version 0 is 0.92 and 0.94. Meta data has serialized HRegionInfo's using Writable
- * serialization, and HRegionInfo's does not contain HTableDescriptors. - version 1 for 0.96+
- * keeps HRegionInfo data structures, but changes the byte[] serialization from Writables to
- * Protobuf. See HRegionInfo.VERSION
+ * The current version of the meta table.
+ * - pre-hbase 0.92. There is no META_VERSION column in the root table
+ * in this case. The meta has HTableDescriptor serialized into the HRegionInfo;
+ * - version 0 is 0.92 and 0.94. Meta data has serialized HRegionInfo's using
+ * Writable serialization, and HRegionInfo's does not contain HTableDescriptors.
+ * - version 1 for 0.96+ keeps HRegionInfo data structures, but changes the
+ * byte[] serialization from Writables to Protobuf.
+ * See HRegionInfo.VERSION
*/
public static final short META_VERSION = 1;
@@ -486,24 +512,25 @@ public final class HConstants {
/**
* An empty instance.
*/
- public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+ public static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(EMPTY_BYTE_ARRAY);
/**
* Used by scanners, etc when they want to start at the beginning of a region
*/
- public static final byte[] EMPTY_START_ROW = EMPTY_BYTE_ARRAY;
+ public static final byte [] EMPTY_START_ROW = EMPTY_BYTE_ARRAY;
/**
* Last row in a table.
*/
- public static final byte[] EMPTY_END_ROW = EMPTY_START_ROW;
+ public static final byte [] EMPTY_END_ROW = EMPTY_START_ROW;
/**
- * Used by scanners and others when they're trying to detect the end of a table
- */
- public static final byte[] LAST_ROW = EMPTY_BYTE_ARRAY;
+ * Used by scanners and others when they're trying to detect the end of a
+ * table
+ */
+ public static final byte [] LAST_ROW = EMPTY_BYTE_ARRAY;
/**
* Max length a row can have because of the limitation in TFile.
@@ -511,8 +538,9 @@ public final class HConstants {
public static final int MAX_ROW_LENGTH = Short.MAX_VALUE;
/**
- * Timestamp to use when we want to refer to the latest cell. This is the timestamp sent by
- * clients when no timestamp is specified on commit.
+ * Timestamp to use when we want to refer to the latest cell.
+ * This is the timestamp sent by clients when no timestamp is specified on
+ * commit.
*/
public static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
@@ -524,12 +552,17 @@ public final class HConstants {
/**
* LATEST_TIMESTAMP in bytes form
*/
- public static final byte[] LATEST_TIMESTAMP_BYTES = {
+ public static final byte [] LATEST_TIMESTAMP_BYTES = {
// big-endian
- (byte) (LATEST_TIMESTAMP >>> 56), (byte) (LATEST_TIMESTAMP >>> 48),
- (byte) (LATEST_TIMESTAMP >>> 40), (byte) (LATEST_TIMESTAMP >>> 32),
- (byte) (LATEST_TIMESTAMP >>> 24), (byte) (LATEST_TIMESTAMP >>> 16),
- (byte) (LATEST_TIMESTAMP >>> 8), (byte) LATEST_TIMESTAMP };
+ (byte) (LATEST_TIMESTAMP >>> 56),
+ (byte) (LATEST_TIMESTAMP >>> 48),
+ (byte) (LATEST_TIMESTAMP >>> 40),
+ (byte) (LATEST_TIMESTAMP >>> 32),
+ (byte) (LATEST_TIMESTAMP >>> 24),
+ (byte) (LATEST_TIMESTAMP >>> 16),
+ (byte) (LATEST_TIMESTAMP >>> 8),
+ (byte) LATEST_TIMESTAMP,
+ };
/**
* Define for 'return-all-versions'.
@@ -539,7 +572,7 @@ public final class HConstants {
/**
* Unlimited time-to-live.
*/
- // public static final int FOREVER = -1;
+// public static final int FOREVER = -1;
public static final int FOREVER = Integer.MAX_VALUE;
/**
@@ -554,10 +587,10 @@ public final class HConstants {
public static final int HOUR_IN_SECONDS = 60 * 60;
public static final int MINUTE_IN_SECONDS = 60;
- // TODO: although the following are referenced widely to format strings for
- // the shell. They really aren't a part of the public API. It would be
- // nice if we could put them somewhere where they did not need to be
- // public. They could have package visibility
+ //TODO: although the following are referenced widely to format strings for
+ // the shell. They really aren't a part of the public API. It would be
+ // nice if we could put them somewhere where they did not need to be
+ // public. They could have package visibility
public static final String NAME = "NAME";
public static final String VERSIONS = "VERSIONS";
public static final String IN_MEMORY = "IN_MEMORY";
@@ -565,38 +598,45 @@ public final class HConstants {
public static final String CONFIGURATION = "CONFIGURATION";
/**
- * Retrying we multiply hbase.client.pause setting by what we have in this array until we run out
- * of array items. Retries beyond this use the last number in the array. So, for example, if
- * hbase.client.pause is 1 second, and maximum retries count hbase.client.retries.number is 10, we
- * will retry at the following intervals: 1, 2, 3, 5, 10, 20, 40, 100, 100, 100. With 100ms, a
- * back-off of 200 means 20s
+ * Retrying we multiply hbase.client.pause setting by what we have in this array until we
+ * run out of array items. Retries beyond this use the last number in the array. So, for
+ * example, if hbase.client.pause is 1 second, and maximum retries count
+ * hbase.client.retries.number is 10, we will retry at the following intervals:
+ * 1, 2, 3, 5, 10, 20, 40, 100, 100, 100.
+ * With 100ms, a back-off of 200 means 20s
*/
- public static final int[] RETRY_BACKOFF =
- { 1, 2, 3, 5, 10, 20, 40, 100, 100, 100, 100, 200, 200 };
+ public static final int [] RETRY_BACKOFF = {1, 2, 3, 5, 10, 20, 40, 100, 100, 100, 100, 200, 200};
public static final String REGION_IMPL = "hbase.hregion.impl";
/** modifyTable op for replacing the table descriptor */
@InterfaceAudience.Private
public static enum Modify {
- CLOSE_REGION, TABLE_COMPACT, TABLE_FLUSH, TABLE_MAJOR_COMPACT, TABLE_SET_HTD, TABLE_SPLIT
+ CLOSE_REGION,
+ TABLE_COMPACT,
+ TABLE_FLUSH,
+ TABLE_MAJOR_COMPACT,
+ TABLE_SET_HTD,
+ TABLE_SPLIT
}
/**
- * Scope tag for locally scoped data. This data will not be replicated.
+ * Scope tag for locally scoped data.
+ * This data will not be replicated.
*/
public static final int REPLICATION_SCOPE_LOCAL = 0;
/**
- * Scope tag for globally scoped data. This data will be replicated to all peers.
+ * Scope tag for globally scoped data.
+ * This data will be replicated to all peers.
*/
public static final int REPLICATION_SCOPE_GLOBAL = 1;
/**
- * Default cluster ID, cannot be used to identify a cluster so a key with this value means it
- * wasn't meant for replication.
+ * Default cluster ID, cannot be used to identify a cluster so a key with
+ * this value means it wasn't meant for replication.
*/
- public static final UUID DEFAULT_CLUSTER_ID = new UUID(0L, 0L);
+ public static final UUID DEFAULT_CLUSTER_ID = new UUID(0L,0L);
/**
* Parameter name for maximum number of bytes returned when calling a scanner's next method.
@@ -613,22 +653,27 @@ public final class HConstants {
"hbase.server.scanner.max.result.size";
/**
- * Maximum number of bytes returned when calling a scanner's next method. Note that when a single
- * row is larger than this limit the row is still returned completely. The default value is 2MB.
+ * Maximum number of bytes returned when calling a scanner's next method.
+ * Note that when a single row is larger than this limit the row is still
+ * returned completely.
+ *
+ * The default value is 2MB.
*/
public static final long DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE = 2 * 1024 * 1024;
/**
- * Maximum number of bytes returned when calling a scanner's next method. Note that when a single
- * row is larger than this limit the row is still returned completely. Safety setting to protect
- * the region server. The default value is 100MB. (a client would rarely request larger chunks on
- * purpose)
+ * Maximum number of bytes returned when calling a scanner's next method.
+ * Note that when a single row is larger than this limit the row is still
+ * returned completely.
+ * Safety setting to protect the region server.
+ *
+ * The default value is 100MB. (a client would rarely request larger chunks on purpose)
*/
public static final long DEFAULT_HBASE_SERVER_SCANNER_MAX_RESULT_SIZE = 100 * 1024 * 1024;
/**
- * Parameter name for client pause value, used mostly as value to wait before running a retry of a
- * failed get, region lookup, etc.
+ * Parameter name for client pause value, used mostly as value to wait
+ * before running a retry of a failed get, region lookup, etc.
*/
public static final String HBASE_CLIENT_PAUSE = "hbase.client.pause";
@@ -648,7 +693,8 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS = 100;
/**
- * The maximum number of concurrent connections the client will maintain to a single RegionServer.
+ * The maximum number of concurrent connections the client will maintain to a single
+ * RegionServer.
*/
public static final String HBASE_CLIENT_MAX_PERSERVER_TASKS = "hbase.client.max.perserver.tasks";
@@ -658,7 +704,8 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS = 2;
/**
- * The maximum number of concurrent connections the client will maintain to a single Region.
+ * The maximum number of concurrent connections the client will maintain to a single
+ * Region.
*/
public static final String HBASE_CLIENT_MAX_PERREGION_TASKS = "hbase.client.max.perregion.tasks";
@@ -668,8 +715,8 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS = 1;
/**
- * Parameter name for server pause value, used mostly as value to wait before running a retry of a
- * failed operation.
+ * Parameter name for server pause value, used mostly as value to wait before
+ * running a retry of a failed operation.
*/
public static final String HBASE_SERVER_PAUSE = "hbase.server.pause";
@@ -679,9 +726,9 @@ public final class HConstants {
public static final int DEFAULT_HBASE_SERVER_PAUSE = 1000;
/**
- * Parameter name for maximum retries, used as maximum for all retryable operations such as
- * fetching of the root region from root region server, getting a cell's value, starting a row
- * update, etc.
+ * Parameter name for maximum retries, used as maximum for all retryable
+ * operations such as fetching of the root region from root region server,
+ * getting a cell's value, starting a row update, etc.
*/
public static final String HBASE_CLIENT_RETRIES_NUMBER = "hbase.client.retries.number";
@@ -701,9 +748,10 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_SCANNER_CACHING = Integer.MAX_VALUE;
/**
- * Parameter name for number of rows that will be fetched when calling next on a scanner if it is
- * not served from memory. Higher caching values will enable faster scanners but will eat up more
- * memory and some calls of next may take longer and longer times when the cache is empty.
+ * Parameter name for number of rows that will be fetched when calling next on
+ * a scanner if it is not served from memory. Higher caching values will
+ * enable faster scanners but will eat up more memory and some calls of next
+ * may take longer and longer times when the cache is empty.
*/
public static final String HBASE_META_SCANNER_CACHING = "hbase.meta.scanner.caching";
@@ -870,13 +918,18 @@ public final class HConstants {
public static final String LOCALHOST = "localhost";
/**
- * If this parameter is set to true, then hbase will read data and then verify checksums. Checksum
- * verification inside hdfs will be switched off. However, if the hbase-checksum verification
- * fails, then it will switch back to using hdfs checksums for verifiying data that is being read
- * from storage. If this parameter is set to false, then hbase will not verify any checksums,
- * instead it will depend on checksum verification being done in the hdfs client.
+ * If this parameter is set to true, then hbase will read
+ * data and then verify checksums. Checksum verification
+ * inside hdfs will be switched off. However, if the hbase-checksum
+ * verification fails, then it will switch back to using
+ * hdfs checksums for verifiying data that is being read from storage.
+ *
+ * If this parameter is set to false, then hbase will not
+ * verify any checksums, instead it will depend on checksum verification
+ * being done in the hdfs client.
*/
- public static final String HBASE_CHECKSUM_VERIFICATION = "hbase.regionserver.checksum.verify";
+ public static final String HBASE_CHECKSUM_VERIFICATION =
+ "hbase.regionserver.checksum.verify";
public static final String LOCALHOST_IP = "127.0.0.1";
@@ -891,15 +944,17 @@ public final class HConstants {
public static final int DEFAULT_REGION_SERVER_HANDLER_COUNT = 30;
/*
- * REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT: -1 => Disable aborting 0 => Abort if even a
- * single handler has died 0.x => Abort only when this percent of handlers have died 1 => Abort
- * only all of the handers have died
+ * REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT:
+ * -1 => Disable aborting
+ * 0 => Abort if even a single handler has died
+ * 0.x => Abort only when this percent of handlers have died
+ * 1 => Abort only all of the handers have died
*/
public static final String REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT =
"hbase.regionserver.handler.abort.on.error.percent";
public static final double DEFAULT_REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT = 0.5;
- // High priority handlers to deal with admin requests and system table operation requests
+ //High priority handlers to deal with admin requests and system table operation requests
public static final String REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT =
"hbase.regionserver.metahandler.count";
public static final int DEFAULT_REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT = 20;
@@ -922,16 +977,19 @@ public final class HConstants {
public static final int DEFAULT_META_REPLICA_NUM = 1;
/**
- * The name of the configuration parameter that specifies the number of bytes in a newly created
- * checksum chunk.
+ * The name of the configuration parameter that specifies
+ * the number of bytes in a newly created checksum chunk.
*/
- public static final String BYTES_PER_CHECKSUM = "hbase.hstore.bytes.per.checksum";
+ public static final String BYTES_PER_CHECKSUM =
+ "hbase.hstore.bytes.per.checksum";
/**
- * The name of the configuration parameter that specifies the name of an algorithm that is used to
- * compute checksums for newly created blocks.
+ * The name of the configuration parameter that specifies
+ * the name of an algorithm that is used to compute checksums
+ * for newly created blocks.
*/
- public static final String CHECKSUM_TYPE_NAME = "hbase.hstore.checksum.algorithm";
+ public static final String CHECKSUM_TYPE_NAME =
+ "hbase.hstore.checksum.algorithm";
/** Enable file permission modification from standard hbase */
public static final String ENABLE_DATA_FILE_UMASK = "hbase.data.umask.enable";
@@ -939,14 +997,16 @@ public final class HConstants {
public static final String DATA_FILE_UMASK_KEY = "hbase.data.umask";
/** Configuration name of WAL Compression */
- public static final String ENABLE_WAL_COMPRESSION = "hbase.regionserver.wal.enablecompression";
-
- /**
- * Configuration name of WAL storage policy Valid values are: NONE: no preference in destination
- * of block replicas ONE_SSD: place only one block replica in SSD and the remaining in default
- * storage and ALL_SSD: place all block replicas on SSD See
- * http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html
- */
+ public static final String ENABLE_WAL_COMPRESSION =
+ "hbase.regionserver.wal.enablecompression";
+
+ /** Configuration name of WAL storage policy
+ * Valid values are:
+ * NONE: no preference in destination of block replicas
+ * ONE_SSD: place only one block replica in SSD and the remaining in default storage
+ * and ALL_SSD: place all block replicas on SSD
+ *
+ * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
public static final String DEFAULT_WAL_STORAGE_POLICY = "NONE";
@@ -957,16 +1017,17 @@ public final class HConstants {
public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
/**
- * The byte array represents for NO_NEXT_INDEXED_KEY; The actual value is irrelevant because this
- * is always compared by reference.
+ * The byte array represents for NO_NEXT_INDEXED_KEY;
+ * The actual value is irrelevant because this is always compared by reference.
*/
public static final Cell NO_NEXT_INDEXED_KEY = new KeyValue();
/** delimiter used between portions of a region name */
public static final int DELIMITER = ',';
/**
- * QOS attributes: these attributes are used to demarcate RPC call processing by different set of
- * handlers. For example, HIGH_QOS tagged methods are handled by high priority handlers.
+ * QOS attributes: these attributes are used to demarcate RPC call processing
+ * by different set of handlers. For example, HIGH_QOS tagged methods are
+ * handled by high priority handlers.
*/
// normal_QOS < QOS_threshold < replication_QOS < replay_QOS < admin_QOS < high_QOS
public static final int NORMAL_QOS = 0;
@@ -981,8 +1042,8 @@ public final class HConstants {
public static final String HFILE_ARCHIVE_DIRECTORY = "archive";
/**
- * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for remaining
- * snapshot constants; this is here to keep HConstants dependencies at a minimum and
+ * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for
+ * remaining snapshot constants; this is here to keep HConstants dependencies at a minimum and
* uni-directional.
*/
public static final String SNAPSHOT_DIR_NAME = ".hbase-snapshot";
@@ -998,42 +1059,49 @@ public final class HConstants {
public static final String REGIONSERVER_METRICS_PERIOD = "hbase.regionserver.metrics.period";
public static final long DEFAULT_REGIONSERVER_METRICS_PERIOD = 5000;
/** Directories that are not HBase table directories */
- public static final List<String> HBASE_NON_TABLE_DIRS = Collections.unmodifiableList(Arrays
- .asList(new String[] { HBCK_SIDELINEDIR_NAME, HBASE_TEMP_DIRECTORY, MIGRATION_NAME }));
+ public static final List<String> HBASE_NON_TABLE_DIRS =
+ Collections.unmodifiableList(Arrays.asList(new String[] {
+ HBCK_SIDELINEDIR_NAME, HBASE_TEMP_DIRECTORY, MIGRATION_NAME
+ }));
/** Directories that are not HBase user table directories */
- public static final List<String> HBASE_NON_USER_TABLE_DIRS = Collections.unmodifiableList(Arrays
- .asList((String[]) ArrayUtils.addAll(
- new String[] { TableName.META_TABLE_NAME.getNameAsString() },
- HBASE_NON_TABLE_DIRS.toArray())));
+ public static final List<String> HBASE_NON_USER_TABLE_DIRS =
+ Collections.unmodifiableList(Arrays.asList((String[])ArrayUtils.addAll(
+ new String[] { TableName.META_TABLE_NAME.getNameAsString() },
+ HBASE_NON_TABLE_DIRS.toArray())));
/** Health script related settings. */
public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location";
public static final String HEALTH_SCRIPT_TIMEOUT = "hbase.node.health.script.timeout";
- public static final String HEALTH_CHORE_WAKE_FREQ = "hbase.node.health.script.frequency";
+ public static final String HEALTH_CHORE_WAKE_FREQ =
+ "hbase.node.health.script.frequency";
public static final long DEFAULT_HEALTH_SCRIPT_TIMEOUT = 60000;
/**
* The maximum number of health check failures a server can encounter consecutively.
*/
- public static final String HEALTH_FAILURE_THRESHOLD = "hbase.node.health.failure.threshold";
+ public static final String HEALTH_FAILURE_THRESHOLD =
+ "hbase.node.health.failure.threshold";
public static final int DEFAULT_HEALTH_FAILURE_THRESHOLD = 3;
+
/**
- * Setting to activate, or not, the publication of the status by the master. Default notification
- * is by a multicast message.
+ * Setting to activate, or not, the publication of the status by the master. Default
+ * notification is by a multicast message.
*/
public static final String STATUS_PUBLISHED = "hbase.status.published";
public static final boolean STATUS_PUBLISHED_DEFAULT = false;
/**
- * IP to use for the multicast status messages between the master and the clients. The default
- * address is chosen as one among others within the ones suitable for multicast messages.
+ * IP to use for the multicast status messages between the master and the clients.
+ * The default address is chosen as one among others within the ones suitable for multicast
+ * messages.
*/
public static final String STATUS_MULTICAST_ADDRESS = "hbase.status.multicast.address.ip";
public static final String DEFAULT_STATUS_MULTICAST_ADDRESS = "226.1.1.3";
/**
- * The address to use for binding the local socket for receiving multicast. Defaults to 0.0.0.0.
+ * The address to use for binding the local socket for receiving multicast. Defaults to
+ * 0.0.0.0.
* @see <a href="https://issues.apache.org/jira/browse/HBASE-9961">HBASE-9961</a>
*/
public static final String STATUS_MULTICAST_BIND_ADDRESS =
@@ -1066,7 +1134,7 @@ public final class HConstants {
/** Configuration key for the name of the alternate master key for the cluster, a string */
public static final String CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY =
- "hbase.crypto.master.alternate.key.name";
+ "hbase.crypto.master.alternate.key.name";
/** Configuration key for the algorithm to use when encrypting the WAL, a string */
public static final String CRYPTO_WAL_ALGORITHM_CONF_KEY = "hbase.crypto.wal.algorithm";
@@ -1098,7 +1166,7 @@ public final class HConstants {
/** Config for pluggable consensus provider */
public static final String HBASE_COORDINATED_STATE_MANAGER_CLASS =
- "hbase.coordinated.state.manager.class";
+ "hbase.coordinated.state.manager.class";
/** Configuration key for SplitLog manager timeout */
public static final String HBASE_SPLITLOG_MANAGER_TIMEOUT = "hbase.splitlog.manager.timeout";
@@ -1112,18 +1180,17 @@ public final class HConstants {
// hbase-common?
/**
- * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path to the
- * file that will host the file-based cache. See BucketCache#getIOEngineFromName() for list of
- * supported ioengine options.
- * <p>
- * Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
+ * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path
+ * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for
+ * list of supported ioengine options.
+ * <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
*/
public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
/**
* When using bucket cache, this is a float that EITHER represents a percentage of total heap
- * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the
- * cache.
+ * memory size to give to the cache (if < 1.0) OR, it is the capacity in
+ * megabytes of the cache.
*/
public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
@@ -1136,25 +1203,26 @@ public final class HConstants {
public static final String HBASE_CLIENT_FAST_FAIL_MODE_ENABLED =
"hbase.client.fast.fail.mode.enabled";
- public static final boolean HBASE_CLIENT_ENABLE_FAST_FAIL_MODE_DEFAULT = false;
+ public static final boolean HBASE_CLIENT_ENABLE_FAST_FAIL_MODE_DEFAULT =
+ false;
public static final String HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS =
"hbase.client.fastfail.threshold";
- public static final long HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT = 60000;
+ public static final long HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT =
+ 60000;
public static final String HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS =
"hbase.client.fast.fail.cleanup.duration";
- public static final long HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT = 600000;
+ public static final long HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT =
+ 600000;
public static final String HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL =
"hbase.client.fast.fail.interceptor.impl";
- /**
- * Config key for if the server should send backpressure and if the client should listen to that
- * backpressure from the server
- */
+ /** Config key for if the server should send backpressure and if the client should listen to
+ * that backpressure from the server */
public static final String ENABLE_CLIENT_BACKPRESSURE = "hbase.client.backpressure.enabled";
public static final boolean DEFAULT_ENABLE_CLIENT_BACKPRESSURE = false;
@@ -1166,11 +1234,11 @@ public final class HConstants {
public static final float DEFAULT_HEAP_OCCUPANCY_HIGH_WATERMARK = 0.98f;
/**
- * The max number of threads used for splitting storefiles in parallel during the region split
- * process.
+ * The max number of threads used for splitting storefiles in parallel during
+ * the region split process.
*/
public static final String REGION_SPLIT_THREADS_MAX =
- "hbase.regionserver.region.split.threads.max";
+ "hbase.regionserver.region.split.threads.max";
/** Canary config keys */
public static final String HBASE_CANARY_WRITE_DATA_TTL_KEY = "hbase.canary.write.data.ttl";
@@ -1195,15 +1263,6 @@ public final class HConstants {
public static final String ZK_SERVER_KEYTAB_FILE = "hbase.zookeeper.server.keytab.file";
public static final String ZK_SERVER_KERBEROS_PRINCIPAL =
"hbase.zookeeper.server.kerberos.principal";
- /**
- * Backup/Restore constants
- */
-
- public final static String BACKUP_ENABLE_KEY = "hbase.backup.enable";
- public final static boolean BACKUP_ENABLE_DEFAULT = true;
- public final static String BACKUP_SYSTEM_TTL_KEY = "hbase.backup.system.ttl";
- // Default TTL = 1 year
- public final static int BACKUP_SYSTEM_TTL_DEFAULT = 365 * 24 * 3600;
private HConstants() {
// Can't be instantiated with this ctor.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 807b021..26aad71 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -394,11 +394,6 @@
<version>${project.version}</version>
<optional>true</optional>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <version>${hadoop-two.version}</version>
- </dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
@@ -412,11 +407,6 @@
<artifactId>commons-collections</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <version>${hadoop-two.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java
deleted file mode 100644
index 28cf627..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Set;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_COMMAND;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * Backup HBase tables locally or on a remote cluster Serve as client entry point for the following
- * features: - Full Backup provide local and remote back/restore for a list of tables - Incremental
- * backup to build on top of full backup as daily/weekly backup - Convert incremental backup WAL
- * files into hfiles - Merge several backup images into one(like merge weekly into monthly) - Add
- * and remove table to and from Backup image - Cancel a backup process - Full backup based on
- * existing snapshot - Describe information of a backup image
- */
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public final class BackupClient {
- private static final Log LOG = LogFactory.getLog(BackupClient.class);
- private static Options opt;
- private static Configuration conf = null;
-
- private BackupClient() {
- throw new AssertionError("Instantiating utility class...");
- }
-
- protected static void init() throws IOException {
- // define supported options
- opt = new Options();
-
- opt.addOption("all", false, "All tables");
- opt.addOption("debug", false, "Enable debug loggings");
- opt.addOption("t", true, "Table name");
-
- // create configuration instance
- conf = getConf();
-
- // disable irrelevant loggers to avoid it mess up command output
- disableUselessLoggers();
-
- }
-
-
- public static void main(String[] args) throws IOException {
- init();
- parseAndRun(args);
- System.exit(0);
- }
-
- /**
- * Set the configuration from a given one.
- * @param newConf A new given configuration
- */
- public synchronized static void setConf(Configuration newConf) {
- conf = newConf;
- BackupUtil.setConf(newConf);
- }
-
- public static Configuration getConf() {
- if (conf == null) {
- conf = BackupUtil.getConf();
- }
- return conf;
- }
-
- private static void disableUselessLoggers() {
- // disable zookeeper log to avoid it mess up command output
- Logger zkLogger = Logger.getLogger("org.apache.zookeeper");
- LOG.debug("Zookeeper log level before set: " + zkLogger.getLevel());
- zkLogger.setLevel(Level.OFF);
- LOG.debug("Zookeeper log level after set: " + zkLogger.getLevel());
-
- // disable hbase zookeeper tool log to avoid it mess up command output
- Logger hbaseZkLogger = Logger.getLogger("org.apache.hadoop.hbase.zookeeper");
- LOG.debug("HBase zookeeper log level before set: " + hbaseZkLogger.getLevel());
- hbaseZkLogger.setLevel(Level.OFF);
- LOG.debug("HBase Zookeeper log level after set: " + hbaseZkLogger.getLevel());
-
- // disable hbase client log to avoid it mess up command output
- Logger hbaseClientLogger = Logger.getLogger("org.apache.hadoop.hbase.client");
- LOG.debug("HBase client log level before set: " + hbaseClientLogger.getLevel());
- hbaseClientLogger.setLevel(Level.OFF);
- LOG.debug("HBase client log level after set: " + hbaseClientLogger.getLevel());
- }
-
- private static void parseAndRun(String[] args) throws IOException {
-
- String cmd = null;
- String[] remainArgs = null;
- if (args == null || args.length == 0) {
- BackupCommands.createCommand(BackupRestoreConstants.BACKUP_COMMAND.HELP, null).execute();
- } else {
- cmd = args[0];
- remainArgs = new String[args.length - 1];
- if (args.length > 1) {
- System.arraycopy(args, 1, remainArgs, 0, args.length - 1);
- }
- }
- CommandLine cmdline = null;
- try {
- cmdline = new PosixParser().parse(opt, remainArgs);
- } catch (ParseException e) {
- LOG.error("Could not parse command", e);
- System.exit(-1);
- }
-
- BACKUP_COMMAND type = BACKUP_COMMAND.HELP;
- if (BACKUP_COMMAND.CREATE.name().equalsIgnoreCase(cmd)) {
- type = BACKUP_COMMAND.CREATE;
- } else if (BACKUP_COMMAND.HELP.name().equalsIgnoreCase(cmd)) {
- type = BACKUP_COMMAND.HELP;
- } else {
- System.out.println("Unsupported command for backup: " + cmd);
- }
-
- // enable debug logging
- Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup");
- if (cmdline.hasOption("debug")) {
- backupClientLogger.setLevel(Level.DEBUG);
- } else {
- backupClientLogger.setLevel(Level.INFO);
- }
-
- BackupCommands.createCommand(type, cmdline).execute();
- }
-
- /**
- * Send backup request to server, and monitor the progress if necessary
- * @param backupType : full or incremental
- * @param backupRootPath : the rooPath specified by user
- * @param tableListStr : the table list specified by user
- * @param snapshot : using existing snapshot if specified by user (in future jira)
- * @return backupId backup id
- * @throws IOException exception
- * @throws KeeperException excpetion
- */
- public static String create(String backupType, String backupRootPath, String tableListStr,
- String snapshot) throws IOException {
-
- String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime();
-
- // check target path first, confirm it doesn't exist before backup
- boolean isTargetExist = false;
- try {
- isTargetExist = HBackupFileSystem.checkPathExist(backupRootPath, conf);
- } catch (IOException e) {
- String expMsg = e.getMessage();
- String newMsg = null;
- if (expMsg.contains("No FileSystem for scheme")) {
- newMsg =
- "Unsupported filesystem scheme found in the backup target url. Error Message: "
- + newMsg;
- LOG.error(newMsg);
- throw new IOException(newMsg);
- } else {
- throw e;
- }
- } catch (RuntimeException e) {
- LOG.error(e.getMessage());
- throw e;
- }
-
- if (isTargetExist) {
- LOG.info("Using existing backup root dir: " + backupRootPath);
- } else {
- LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");
- }
-
- // table list specified for backup, trigger backup on specified tables
- String tableList = tableListStr;
- // (tableListStr == null) ? null : tableListStr.replaceAll(
- // BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND,
- // BackupRestoreConstants.TABLENAME_DELIMITER_IN_ZNODE);
- try {
- requestBackup(backupId, backupType, tableList, backupRootPath, snapshot);
- } catch (RuntimeException e) {
- String errMsg = e.getMessage();
- if (errMsg != null
- && (errMsg.startsWith("Non-existing tables found") || errMsg
- .startsWith("Snapshot is not found"))) {
- LOG.error(errMsg + ", please check your command");
- throw e;
- } else {
- throw e;
- }
- }
- return backupId;
- }
-
- /**
- * Prepare and submit Backup request
- * @param backupId : backup_timestame (something like backup_1398729212626)
- * @param backupType : full or incremental
- * @param tableList : tables to be backuped
- * @param targetRootDir : specified by user
- * @param snapshot : use existing snapshot if specified by user (for future jira)
- * @throws IOException exception
- */
- protected static void requestBackup(String backupId, String backupType, String tableList,
- String targetRootDir, String snapshot) throws IOException {
-
- Configuration conf = getConf();
- BackupManager backupManager = null;
- BackupContext backupContext = null;
- if (snapshot != null) {
- LOG.warn("Snapshot option specified, backup type and table option will be ignored,\n"
- + "full backup will be taken based on the given snapshot.");
- throw new IOException("backup using existing Snapshot will be implemented in future jira");
- }
-
- HBaseAdmin hbadmin = null;
- Connection conn = null;
- try {
-
- backupManager = new BackupManager(conf);
- String tables = tableList;
- if (backupType.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- Set<String> incrTableSet = backupManager.getIncrementalBackupTableSet();
- if (incrTableSet.isEmpty()) {
- LOG.warn("Incremental backup table set contains no table.\n"
- + "Use 'backup create full' or 'backup stop' to \n "
- + "change the tables covered by incremental backup.");
- throw new RuntimeException("No table covered by incremental backup.");
- }
- StringBuilder sb = new StringBuilder();
- for (String tableName : incrTableSet) {
- sb.append(tableName + " ");
- }
- LOG.info("Incremental backup for the following table set: " + sb.toString());
- tables =
- sb.toString().trim()
- .replaceAll(" ", BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- }
-
- // check whether table exists first before starting real request
- if (tables != null) {
- String[] tableNames = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- ArrayList<String> noneExistingTableList = null;
- conn = ConnectionFactory.createConnection(conf);
- hbadmin = (HBaseAdmin) conn.getAdmin();
- for (String tableName : tableNames) {
- if (!hbadmin.tableExists(TableName.valueOf(tableName))) {
- if (noneExistingTableList == null) {
- noneExistingTableList = new ArrayList<String>();
- }
- noneExistingTableList.add(tableName);
- }
- }
- if (noneExistingTableList != null) {
- if (backupType.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- LOG.warn("Incremental backup table set contains no-exising table: "
- + noneExistingTableList);
- } else {
- // Throw exception only in full mode - we try to backup non-existing table
- throw new RuntimeException("Non-existing tables found in the table list: "
- + noneExistingTableList);
- }
- }
- }
-
- // if any target table backup dir already exist, then no backup action taken
- String[] tableNames = null;
- if (tables != null && !tables.equals("")) {
- tableNames = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- }
- if (tableNames != null && tableNames.length > 0) {
- for (String table : tableNames) {
- String targetTableBackupDir =
- HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
- Path targetTableBackupDirPath = new Path(targetTableBackupDir);
- FileSystem outputFs = FileSystem.get(targetTableBackupDirPath.toUri(), conf);
- if (outputFs.exists(targetTableBackupDirPath)) {
- throw new IOException("Target backup directory " + targetTableBackupDir
- + " exists already.");
- }
- }
- }
- backupContext =
- backupManager.createBackupContext(backupId, backupType, tables, targetRootDir, snapshot);
- backupManager.initialize();
- backupManager.dispatchRequest(backupContext);
- } catch (BackupException e) {
- // suppress the backup exception wrapped within #initialize or #dispatchRequest, backup
- // exception has already been handled normally
- StackTraceElement[] stes = e.getStackTrace();
- for (StackTraceElement ste : stes) {
- LOG.info(ste);
- }
- LOG.error("Backup Exception " + e.getMessage());
- } finally {
- if (hbadmin != null) {
- hbadmin.close();
- }
- if (conn != null) {
- conn.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java
deleted file mode 100644
index c7aff15..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_COMMAND;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * General backup commands, options and usage messages
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-final class BackupCommands {
-
- private static final String USAGE = "Usage: hbase backup COMMAND\n"
- + "where COMMAND is one of:\n" + " create create a new backup image\n"
- + "Enter \'help COMMAND\' to see help message for each command\n";
-
- private static final String CREATE_CMD_USAGE =
- "Usage: hbase backup create <type> <backup_root_path> [tables] [-s name] [-convert] "
- + "[-silent]\n" + " type \"full\" to create a full backup image;\n"
- + " \"incremental\" to create an incremental backup image\n"
- + " backup_root_path The full root path to store the backup image,\n"
- + " the prefix can be gpfs, hdfs or webhdfs\n" + " Options:\n"
- + " tables If no tables (\"\") are specified, all tables are backed up. "
- + "Otherwise it is a\n" + " comma separated list of tables.\n"
- + " -s name Use the specified snapshot for full backup\n"
- + " -convert For an incremental backup, convert WAL files to HFiles\n";
-
- interface Command {
- void execute() throws IOException;
- }
-
- private BackupCommands() {
- throw new AssertionError("Instantiating utility class...");
- }
-
- static Command createCommand(BACKUP_COMMAND type, CommandLine cmdline) {
- Command cmd = null;
- switch (type) {
- case CREATE:
- cmd = new CreateCommand(cmdline);
- break;
- case HELP:
- default:
- cmd = new HelpCommand(cmdline);
- break;
- }
- return cmd;
- }
-
- private static class CreateCommand implements Command {
- CommandLine cmdline;
-
- CreateCommand(CommandLine cmdline) {
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null) {
- System.out.println("ERROR: missing arguments");
- System.out.println(CREATE_CMD_USAGE);
- System.exit(-1);
- }
- String[] args = cmdline.getArgs();
- if (args.length < 2 || args.length > 3) {
- System.out.println("ERROR: wrong number of arguments");
- System.out.println(CREATE_CMD_USAGE);
- System.exit(-1);
- }
-
- if (!BackupRestoreConstants.BACKUP_TYPE_FULL.equalsIgnoreCase(args[0])
- && !BackupRestoreConstants.BACKUP_TYPE_INCR.equalsIgnoreCase(args[0])) {
- System.out.println("ERROR: invalid backup type");
- System.out.println(CREATE_CMD_USAGE);
- System.exit(-1);
- }
-
- String snapshot = cmdline.hasOption('s') ? cmdline.getOptionValue('s') : null;
- String tables = (args.length == 3) ? args[2] : null;
-
- try {
- BackupClient.create(args[0], args[1], tables, snapshot);
- } catch (RuntimeException e) {
- System.out.println("ERROR: " + e.getMessage());
- System.exit(-1);
- }
- }
- }
-
- private static class HelpCommand implements Command {
- CommandLine cmdline;
-
- HelpCommand(CommandLine cmdline) {
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null) {
- System.out.println(USAGE);
- System.exit(0);
- }
-
- String[] args = cmdline.getArgs();
- if (args == null || args.length == 0) {
- System.out.println(USAGE);
- System.exit(0);
- }
-
- if (args.length != 1) {
- System.out.println("Only support check help message of a single command type");
- System.out.println(USAGE);
- System.exit(0);
- }
-
- String type = args[0];
-
- if (BACKUP_COMMAND.CREATE.name().equalsIgnoreCase(type)) {
- System.out.println(CREATE_CMD_USAGE);
- } // other commands will be supported in future jira
- System.exit(0);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java
deleted file mode 100644
index e8d6444..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java
+++ /dev/null
@@ -1,318 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * An object to encapsulate the information for each backup request
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupContext implements Serializable {
-
- private static final long serialVersionUID = 2401435114454300992L;
-
- // backup id: a timestamp when we request the backup
- private String backupId;
-
- // backup type, full or incremental
- private String type;
-
- // target root directory for storing the backup files
- private String targetRootDir;
-
- // overall backup status
- private BackupHandler.BACKUPSTATUS flag;
-
- // overall backup phase
- private BackupHandler.BACKUPPHASE phase;
-
- // overall backup failure message
- private String failedMsg;
-
- // backup status map for all tables
- private Map<String, BackupStatus> backupStatusMap;
-
- // actual start timestamp of the backup process
- private long startTs;
-
- // actual end timestamp of the backup process, could be fail or complete
- private long endTs;
-
- // the total bytes of incremental logs copied
- private long totalBytesCopied;
-
- // for incremental backup, the location of the backed-up hlogs
- private String hlogTargetDir = null;
-
- // incremental backup file list
- transient private List<String> incrBackupFileList;
-
- // new region server log timestamps for table set after distributed log roll
- // key - table name, value - map of RegionServer hostname -> last log rolled timestamp
- transient private HashMap<String, HashMap<String, String>> tableSetTimestampMap;
-
- // cancel flag
- private boolean cancelled = false;
- // backup progress string
-
- private String progress;
-
- public BackupContext() {
- }
-
- public BackupContext(String backupId, String type, String[] tables, String targetRootDir,
- String snapshot) {
- super();
-
- if (backupStatusMap == null) {
- backupStatusMap = new HashMap<String, BackupStatus>();
- }
-
- this.backupId = backupId;
- this.type = type;
- this.targetRootDir = targetRootDir;
-
- this.addTables(tables);
-
- if (type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
- setHlogTargetDir(HBackupFileSystem.getLogBackupDir(targetRootDir, backupId));
- }
-
- this.startTs = 0;
- this.endTs = 0;
-
- }
-
- /**
- * Set progress string
- * @param msg progress message
- */
-
- public void setProgress(String msg) {
- this.progress = msg;
- }
-
- /**
- * Get current progress msg
- */
- public String getProgress() {
- return progress;
- }
-
- /**
- * Mark cancel flag.
- */
- public void markCancel() {
- this.cancelled = true;
- }
-
- /**
- * Has been marked as cancelled or not.
- * @return True if marked as cancelled
- */
- public boolean isCancelled() {
- return this.cancelled;
- }
-
- public String getBackupId() {
- return backupId;
- }
-
- public void setBackupId(String backupId) {
- this.backupId = backupId;
- }
-
- public BackupStatus getBackupStatus(String table) {
- return this.backupStatusMap.get(table);
- }
-
- public String getFailedMsg() {
- return failedMsg;
- }
-
- public void setFailedMsg(String failedMsg) {
- this.failedMsg = failedMsg;
- }
-
- public long getStartTs() {
- return startTs;
- }
-
- public void setStartTs(long startTs) {
- this.startTs = startTs;
- }
-
- public long getEndTs() {
- return endTs;
- }
-
- public void setEndTs(long endTs) {
- this.endTs = endTs;
- }
-
- public long getTotalBytesCopied() {
- return totalBytesCopied;
- }
-
- public BackupHandler.BACKUPSTATUS getFlag() {
- return flag;
- }
-
- public void setFlag(BackupHandler.BACKUPSTATUS flag) {
- this.flag = flag;
- }
-
- public BackupHandler.BACKUPPHASE getPhase() {
- return phase;
- }
-
- public void setPhase(BackupHandler.BACKUPPHASE phase) {
- this.phase = phase;
- }
-
- public String getType() {
- return type;
- }
-
- public void setSnapshotName(String table, String snapshotName) {
- this.backupStatusMap.get(table).setSnapshotName(snapshotName);
- }
-
- public String getSnapshotName(String table) {
- return this.backupStatusMap.get(table).getSnapshotName();
- }
-
- public List<String> getSnapshotNames() {
- List<String> snapshotNames = new ArrayList<String>();
- for (BackupStatus backupStatus : this.backupStatusMap.values()) {
- snapshotNames.add(backupStatus.getSnapshotName());
- }
- return snapshotNames;
- }
-
- public Set<String> getTables() {
- return this.backupStatusMap.keySet();
- }
-
- public String getTableListAsString() {
- return BackupUtil.concat(backupStatusMap.keySet(), ";");
- }
-
- public void addTables(String[] tables) {
- for (String table : tables) {
- BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId);
- this.backupStatusMap.put(table, backupStatus);
- }
- }
-
- public String getTargetRootDir() {
- return targetRootDir;
- }
-
- public void setHlogTargetDir(String hlogTagetDir) {
- this.hlogTargetDir = hlogTagetDir;
- }
-
- public String getHLogTargetDir() {
- return hlogTargetDir;
- }
-
- public List<String> getIncrBackupFileList() {
- return incrBackupFileList;
- }
-
- public List<String> setIncrBackupFileList(List<String> incrBackupFileList) {
- this.incrBackupFileList = incrBackupFileList;
- return this.incrBackupFileList;
- }
-
- /**
- * Set the new region server log timestamps after distributed log roll
- * @param newTableSetTimestampMap table timestamp map
- */
- public void setIncrTimestampMap(HashMap<String,
- HashMap<String, String>> newTableSetTimestampMap) {
- this.tableSetTimestampMap = newTableSetTimestampMap;
- }
-
- /**
- * Get new region server log timestamps after distributed log roll
- * @return new region server log timestamps
- */
- public HashMap<String, HashMap<String, String>> getIncrTimestampMap() {
- return this.tableSetTimestampMap;
- }
-
- /**
- * Get existing snapshot if backing up from existing snapshot.
- * @return The existing snapshot, null if not backing up from existing snapshot
- */
- public String getExistingSnapshot() {
- // this feature will be supported in another Jira
- return null;
- }
-
- /**
- * Check whether this backup context are for backing up from existing snapshot or not.
- * @return true if it is for backing up from existing snapshot, otherwise false
- */
- public boolean fromExistingSnapshot() {
- // this feature will be supported in later jiras
- return false;
- }
-
- public String getTableBySnapshot(String snapshotName) {
- for (Entry<String, BackupStatus> entry : this.backupStatusMap.entrySet()) {
- if (snapshotName.equals(entry.getValue().getSnapshotName())) {
- return entry.getKey();
- }
- }
- return null;
- }
-
- public byte[] toByteArray() throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(baos);
- oos.writeObject(this);
- return baos.toByteArray();
- }
-
- public static BackupContext fromByteArray(byte[] data)
- throws IOException, ClassNotFoundException {
- ByteArrayInputStream bais = new ByteArrayInputStream(data);
- ObjectInputStream ois = new ObjectInputStream(bais);
- return (BackupContext) ois.readObject();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java
deleted file mode 100644
index 223746d..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface BackupCopyService extends Configurable {
- static enum Type {
- FULL, INCREMENTAL
- }
-
- public int copy(BackupHandler backupHandler, Configuration conf, BackupCopyService.Type copyType,
- String[] options) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java
deleted file mode 100644
index fe0d0e2..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * Backup exception
- */
-@SuppressWarnings("serial")
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupException extends HBaseIOException {
- private BackupContext description;
-
- /**
- * Some exception happened for a backup and don't even know the backup that it was about
- * @param msg Full description of the failure
- */
- public BackupException(String msg) {
- super(msg);
- }
-
- /**
- * Some exception happened for a backup with a cause
- * @param cause the cause
- */
- public BackupException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Exception for the given backup that has no previous root cause
- * @param msg reason why the backup failed
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, BackupContext desc) {
- super(msg);
- this.description = desc;
- }
-
- /**
- * Exception for the given backup due to another exception
- * @param msg reason why the backup failed
- * @param cause root cause of the failure
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, Throwable cause, BackupContext desc) {
- super(msg, cause);
- this.description = desc;
- }
-
- /**
- * Exception when the description of the backup cannot be determined, due to some other root
- * cause
- * @param message description of what caused the failure
- * @param e root cause
- */
- public BackupException(String message, Exception e) {
- super(message, e);
- }
-
- public BackupContext getBackupContext() {
- return this.description;
- }
-
-}
[04/29] hbase git commit: HBASE-14030 HBase Backup/Restore Phase 1
(Vladimir Rodionov)
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java
new file mode 100644
index 0000000..14769f9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java
@@ -0,0 +1,642 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
+import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+
+/**
+ * This class provides 'hbase:backup' table API
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class BackupSystemTable {
+
+ private static final Log LOG = LogFactory.getLog(BackupSystemTable.class);
+ private final static String TABLE_NAMESPACE = "hbase";
+ private final static String TABLE_NAME = "backup";
+ private final static TableName tableName = TableName.valueOf(TABLE_NAMESPACE, TABLE_NAME);
+ public final static byte[] familyName = "f".getBytes();
+
+ // Connection to HBase cluster
+ private static Connection connection;
+ // Cluster configuration
+ private static Configuration config;
+ // singleton
+ private static BackupSystemTable table;
+
+ /**
+ * Get instance by a given configuration
+ * @param conf - HBase configuration
+ * @return instance of BackupSystemTable
+ * @throws IOException exception
+ */
+ public synchronized static BackupSystemTable getTable(Configuration conf) throws IOException {
+ if (connection == null) {
+ connection = ConnectionFactory.createConnection(conf);
+ config = conf;
+ // Verify hbase:system exists
+ createSystemTableIfNotExists();
+ table = new BackupSystemTable();
+ }
+ return table;
+ }
+
+ /**
+ * TODO: refactor
+ * @throws IOException exception
+ */
+ public static void close() throws IOException {
+ connection.close();
+ table = null;
+ }
+
+ /**
+ * Gets table name
+ * @return table name
+ */
+ public static TableName getTableName() {
+ return tableName;
+ }
+
+ private static void createSystemTableIfNotExists() throws IOException {
+ Admin admin = null;
+ try {
+ admin = connection.getAdmin();
+ if (admin.tableExists(tableName) == false) {
+ HTableDescriptor tableDesc = new HTableDescriptor(tableName);
+ HColumnDescriptor colDesc = new HColumnDescriptor(familyName);
+ colDesc.setMaxVersions(1);
+ int ttl =
+ config.getInt(HConstants.BACKUP_SYSTEM_TTL_KEY, HConstants.BACKUP_SYSTEM_TTL_DEFAULT);
+ colDesc.setTimeToLive(ttl);
+ tableDesc.addFamily(colDesc);
+ admin.createTable(tableDesc);
+ }
+ } catch (IOException e) {
+ LOG.error(e);
+ throw e;
+ } finally {
+ if (admin != null) {
+ admin.close();
+ }
+ }
+ }
+
+ private BackupSystemTable() {
+ }
+
+ /**
+ * Updates status (state) of a backup session in hbase:backup table
+ * @param context context
+ * @throws IOException exception
+ */
+ public void updateBackupStatus(BackupContext context) throws IOException {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("update backup status in hbase:backup for: " + context.getBackupId()
+ + " set status=" + context.getFlag());
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Put put = BackupSystemTableHelper.createPutForBackupContext(context);
+ table.put(put);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Deletes backup status from hbase:backup table
+ * @param backupId backup id
+ * @throws IOException exception
+ */
+
+ public void deleteBackupStatus(String backupId) throws IOException {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("delete backup status in hbase:backup for " + backupId);
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Delete del = BackupSystemTableHelper.createDeletForBackupContext(backupId);
+ table.delete(del);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Reads backup status object (instance of BackupContext) from hbase:backup table
+ * @param backupId - backupId
+ * @return Current status of backup session or null
+ */
+
+ public BackupContext readBackupStatus(String backupId) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("read backup status from hbase:backup for: " + backupId);
+ }
+
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Get get = BackupSystemTableHelper.createGetForBackupContext(backupId);
+ Result res = table.get(get);
+ if(res.isEmpty()){
+ return null;
+ }
+ return BackupSystemTableHelper.resultToBackupContext(res);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Read the last backup start code (timestamp) of last successful backup. Will return null if
+ * there is no start code stored on hbase or the value is of length 0. These two cases indicate
+ * there is no successful backup completed so far.
+ * @return the timestamp of last successful backup
+ * @throws IOException exception
+ */
+ public String readBackupStartCode() throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("read backup start code from hbase:backup");
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Get get = BackupSystemTableHelper.createGetForStartCode();
+ Result res = table.get(get);
+ if (res.isEmpty()){
+ return null;
+ }
+ Cell cell = res.listCells().get(0);
+ byte[] val = CellUtil.cloneValue(cell);
+ if (val.length == 0){
+ return null;
+ }
+ return new String(val);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte.
+ * @param startCode start code
+ * @throws IOException exception
+ */
+ public void writeBackupStartCode(String startCode) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("write backup start code to hbase:backup " + startCode);
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Put put = BackupSystemTableHelper.createPutForStartCode(startCode);
+ table.put(put);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Get the Region Servers log information after the last log roll from hbase:backup.
+ * @return RS log info
+ * @throws IOException exception
+ */
+ public HashMap<String, String> readRegionServerLastLogRollResult()
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("read region server last roll log result to hbase:backup");
+ }
+ Table table = null;
+ ResultScanner scanner = null;
+
+ try {
+ table = connection.getTable(tableName);
+ Scan scan = BackupSystemTableHelper.createScanForReadRegionServerLastLogRollResult();
+ scan.setMaxVersions(1);
+ scanner = table.getScanner(scan);
+ Result res = null;
+ HashMap<String, String> rsTimestampMap = new HashMap<String, String>();
+ while ((res = scanner.next()) != null) {
+ res.advance();
+ Cell cell = res.current();
+ byte[] row = CellUtil.cloneRow(cell);
+ String server =
+ BackupSystemTableHelper.getServerNameForReadRegionServerLastLogRollResult(row);
+
+ byte[] data = CellUtil.cloneValue(cell);
+ rsTimestampMap.put(server, new String(data));
+ }
+ return rsTimestampMap;
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Writes Region Server last roll log result (timestamp) to hbase:backup table
+ * @param server - Region Server name
+ * @param fileName - last log timestamp
+ * @throws IOException exception
+ */
+ public void writeRegionServerLastLogRollResult(String server, String fileName)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("write region server last roll log result to hbase:backup");
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Put put =
+ BackupSystemTableHelper.createPutForRegionServerLastLogRollResult(server, fileName);
+ table.put(put);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Get all completed backup information (in desc order by time)
+ * @return history info of BackupCompleteData
+ * @throws IOException exception
+ */
+ public ArrayList<BackupCompleteData> getBackupHistory() throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("get backup history from hbase:backup");
+ }
+ Table table = null;
+ ResultScanner scanner = null;
+ ArrayList<BackupCompleteData> list = new ArrayList<BackupCompleteData>();
+ try {
+ table = connection.getTable(tableName);
+ Scan scan = BackupSystemTableHelper.createScanForBackupHistory();
+ scan.setMaxVersions(1);
+ scanner = table.getScanner(scan);
+ Result res = null;
+ while ((res = scanner.next()) != null) {
+ res.advance();
+ BackupContext context = BackupSystemTableHelper.cellToBackupContext(res.current());
+ if (context.getFlag() != BACKUPSTATUS.COMPLETE) {
+ continue;
+ }
+
+ BackupCompleteData history = new BackupCompleteData();
+ history.setBackupToken(context.getBackupId());
+ history.setStartTime(Long.toString(context.getStartTs()));
+ history.setEndTime(Long.toString(context.getEndTs()));
+ history.setBackupRootPath(context.getTargetRootDir());
+ history.setTableList(context.getTableListAsString());
+ history.setType(context.getType());
+ history.setBytesCopied(Long.toString(context.getTotalBytesCopied()));
+
+ if (context.fromExistingSnapshot()) {
+ history.markFromExistingSnapshot();
+ }
+ list.add(history);
+ }
+ return BackupUtil.sortHistoryListDesc(list);
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Get all backup session with a given status (in desc order by time)
+ * @param status status
+ * @return history info of backup contexts
+ * @throws IOException exception
+ */
+ public ArrayList<BackupContext> getBackupContexts(BACKUPSTATUS status) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("get backup contexts from hbase:backup");
+ }
+ Table table = null;
+ ResultScanner scanner = null;
+ ArrayList<BackupContext> list = new ArrayList<BackupContext>();
+ try {
+ table = connection.getTable(tableName);
+ Scan scan = BackupSystemTableHelper.createScanForBackupHistory();
+ scan.setMaxVersions(1);
+ scanner = table.getScanner(scan);
+ Result res = null;
+ while ((res = scanner.next()) != null) {
+ res.advance();
+ BackupContext context = BackupSystemTableHelper.cellToBackupContext(res.current());
+ if (context.getFlag() != status){
+ continue;
+ }
+ list.add(context);
+ }
+ return list;
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Write the current timestamps for each regionserver to hbase:backup after a successful full or
+ * incremental backup. The saved timestamp is of the last log file that was backed up already.
+ * @param tables tables
+ * @param newTimestamps timestamps
+ * @throws IOException exception
+ */
+ public void writeRegionServerLogTimestamp(Set<String> tables,
+ HashMap<String, String> newTimestamps) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("write RS log ts to HBASE_BACKUP");
+ }
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<String, String> entry : newTimestamps.entrySet()) {
+ String host = entry.getKey();
+ String timestamp = entry.getValue();
+ sb.append(host).append(BackupUtil.FIELD_SEPARATOR).append(timestamp)
+ .append(BackupUtil.RECORD_SEPARATOR);
+ }
+ String smap = sb.toString();
+ List<Put> puts = new ArrayList<Put>();
+ for (String table : tables) {
+ Put put = BackupSystemTableHelper.createPutForWriteRegionServerLogTimestamp(table, smap);
+ puts.add(put);
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ table.put(puts);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Read the timestamp for each region server log after the last successful backup. Each table has
+ * its own set of the timestamps. The info is stored for each table as a concatenated string of
+ * rs->timestapmp
+ * @return the timestamp for each region server. key: tableName value:
+ * RegionServer,PreviousTimeStamp
+ * @throws IOException exception
+ */
+ public HashMap<String, HashMap<String, String>> readLogTimestampMap() throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("read RS log ts from HBASE_BACKUP");
+ }
+
+ Table table = null;
+ ResultScanner scanner = null;
+ HashMap<String, HashMap<String, String>> tableTimestampMap =
+ new HashMap<String, HashMap<String, String>>();
+
+ try {
+ table = connection.getTable(tableName);
+ Scan scan = BackupSystemTableHelper.createScanForReadLogTimestampMap();
+ scanner = table.getScanner(scan);
+ Result res = null;
+ while ((res = scanner.next()) != null) {
+ res.advance();
+ Cell cell = res.current();
+ byte[] row = CellUtil.cloneRow(cell);
+ String tabName = BackupSystemTableHelper.getTableNameForReadLogTimestampMap(row);
+ HashMap<String, String> lastBackup = new HashMap<String, String>();
+ byte[] data = CellUtil.cloneValue(cell);
+ if (data == null) {
+ // TODO
+ throw new IOException("Data of last backup data from HBASE_BACKUP "
+ + "is empty. Create a backup first.");
+ }
+ if (data != null && data.length > 0) {
+ String s = new String(data);
+ String[] records = s.split(BackupUtil.RECORD_SEPARATOR);
+ for (String record : records) {
+ String[] flds = record.split(BackupUtil.FIELD_SEPARATOR);
+ if (flds.length != 2) {
+ throw new IOException("data from HBASE_BACKUP is corrupted: "
+ + Arrays.toString(flds));
+ }
+ lastBackup.put(flds[0], flds[1]);
+ }
+ tableTimestampMap.put(tabName, lastBackup);
+ }
+ }
+ return tableTimestampMap;
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Return the current tables covered by incremental backup.
+ * @return set of tableNames
+ * @throws IOException exception
+ */
+ public Set<String> getIncrementalBackupTableSet() throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("get incr backup table set from hbase:backup");
+ }
+ Table table = null;
+ TreeSet<String> set = new TreeSet<String>();
+
+ try {
+ table = connection.getTable(tableName);
+ Get get = BackupSystemTableHelper.createGetForIncrBackupTableSet();
+ Result res = table.get(get);
+ if (res.isEmpty()) {
+ return set;
+ }
+ List<Cell> cells = res.listCells();
+ for (Cell cell : cells) {
+ // qualifier = table name - we use table names as qualifiers
+ // TODO ns:table as qualifier?
+ set.add(new String(CellUtil.cloneQualifier(cell)));
+ }
+ return set;
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Add tables to global incremental backup set
+ * @param tables - set of tables
+ * @throws IOException exception
+ */
+ public void addIncrementalBackupTableSet(Set<String> tables) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("add incr backup table set to hbase:backup");
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Put put = BackupSystemTableHelper.createPutForIncrBackupTableSet(tables);
+ table.put(put);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Register WAL files as eligible for deletion
+ * @param files files
+ * @throws IOException exception
+ */
+ public void addWALFiles(List<String> files, String backupId) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("add WAL files to hbase:backup");
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ List<Put> puts = BackupSystemTableHelper.createPutsForAddWALFiles(files, backupId);
+ table.put(puts);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Check if WAL file is eligible for deletion
+ * @param file file
+ * @return true, if - yes.
+ * @throws IOException exception
+ */
+ public boolean checkWALFile(String file) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Check if WAL file has been already backuped in hbase:backup");
+ }
+ Table table = null;
+ try {
+ table = connection.getTable(tableName);
+ Get get = BackupSystemTableHelper.createGetForCheckWALFile(file);
+ Result res = table.get(get);
+ if (res.isEmpty()){
+ return false;
+ }
+ return true;
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ /**
+ * Checks if we have at least one backup session in hbase:backup This API is used by
+ * BackupLogCleaner
+ * @return true, if - at least one session exists in hbase:backup table
+ * @throws IOException exception
+ */
+ public boolean hasBackupSessions() throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("has backup sessions from hbase:backup");
+ }
+ Table table = null;
+ ResultScanner scanner = null;
+ boolean result = false;
+ try {
+ table = connection.getTable(tableName);
+ Scan scan = BackupSystemTableHelper.createScanForBackupHistory();
+ scan.setMaxVersions(1);
+ scan.setCaching(1);
+ scanner = table.getScanner(scan);
+ if (scanner.next() != null) {
+ result = true;
+ }
+ return result;
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java
new file mode 100644
index 0000000..bf62a84
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+
+
+/**
+ * A collection for methods used by BackupSystemTable.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class BackupSystemTableHelper {
+
+ /**
+ * hbase:backup schema:
+ * 1. Backup sessions rowkey= "session." + backupId; value = serialized
+ * BackupContext
+ * 2. Backup start code rowkey = "startcode"; value = startcode
+ * 3. Incremental backup set rowkey="incrbackupset"; value=[list of tables]
+ * 4. Table-RS-timestamp map rowkey="trslm"+ table_name; value = map[RS-> last WAL timestamp]
+ * 5. RS - WAL ts map rowkey="rslogts."+server; value = last WAL timestamp
+ * 6. WALs recorded rowkey="wals."+WAL unique file name; value = NULL (value is not used)
+ */
+ private static final Log LOG = LogFactory.getLog(BackupSystemTableHelper.class);
+
+ private final static String BACKUP_CONTEXT_PREFIX = "session.";
+ private final static String START_CODE_ROW = "startcode";
+ private final static String INCR_BACKUP_SET = "incrbackupset";
+ private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm.";
+ private final static String RS_LOG_TS_PREFIX = "rslogts.";
+ private final static String WALS_PREFIX = "wals.";
+
+ private final static byte[] q0 = "0".getBytes();
+ private final static byte[] EMPTY_VALUE = new byte[] {};
+
+ private BackupSystemTableHelper() {
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ /**
+ * Creates Put operation for a given backup context object
+ * @param context backup context
+ * @return put operation
+ * @throws IOException exception
+ */
+ static Put createPutForBackupContext(BackupContext context) throws IOException {
+
+ Put put = new Put((BACKUP_CONTEXT_PREFIX + context.getBackupId()).getBytes());
+ put.addColumn(BackupSystemTable.familyName, q0, context.toByteArray());
+ return put;
+ }
+
+ /**
+ * Creates Get operation for a given backup id
+ * @param backupId - backup's ID
+ * @return get operation
+ * @throws IOException exception
+ */
+ static Get createGetForBackupContext(String backupId) throws IOException {
+ Get get = new Get((BACKUP_CONTEXT_PREFIX + backupId).getBytes());
+ get.addFamily(BackupSystemTable.familyName);
+ get.setMaxVersions(1);
+ return get;
+ }
+
+ /**
+ * Creates Delete operation for a given backup id
+ * @param backupId - backup's ID
+ * @return delete operation
+ * @throws IOException exception
+ */
+ public static Delete createDeletForBackupContext(String backupId) {
+ Delete del = new Delete((BACKUP_CONTEXT_PREFIX + backupId).getBytes());
+ del.addFamily(BackupSystemTable.familyName);
+ return del;
+ }
+
+ /**
+ * Converts Result to BackupContext
+ * @param res - HBase result
+ * @return backup context instance
+ * @throws IOException exception
+ */
+ static BackupContext resultToBackupContext(Result res) throws IOException {
+ res.advance();
+ Cell cell = res.current();
+ return cellToBackupContext(cell);
+ }
+
+ /**
+ * Creates Get operation to retrieve start code from hbase:backup
+ * @return get operation
+ * @throws IOException exception
+ */
+ static Get createGetForStartCode() throws IOException {
+ Get get = new Get(START_CODE_ROW.getBytes());
+ get.addFamily(BackupSystemTable.familyName);
+ get.setMaxVersions(1);
+ return get;
+ }
+
+ /**
+ * Creates Put operation to store start code to hbase:backup
+ * @return put operation
+ * @throws IOException exception
+ */
+ static Put createPutForStartCode(String startCode) {
+ Put put = new Put(START_CODE_ROW.getBytes());
+ put.addColumn(BackupSystemTable.familyName, q0, startCode.getBytes());
+ return put;
+ }
+
+ /**
+ * Creates Get to retrieve incremental backup table set from hbase:backup
+ * @return get operation
+ * @throws IOException exception
+ */
+ static Get createGetForIncrBackupTableSet() throws IOException {
+ Get get = new Get(INCR_BACKUP_SET.getBytes());
+ get.addFamily(BackupSystemTable.familyName);
+ get.setMaxVersions(1);
+ return get;
+ }
+
+ /**
+ * Creates Put to store incremental backup table set
+ * @param tables tables
+ * @return put operation
+ */
+ static Put createPutForIncrBackupTableSet(Set<String> tables) {
+ Put put = new Put(INCR_BACKUP_SET.getBytes());
+ for (String table : tables) {
+ put.addColumn(BackupSystemTable.familyName, table.getBytes(), EMPTY_VALUE);
+ }
+ return put;
+ }
+
+ /**
+ * Creates Scan operation to load backup history
+ * @return scan operation
+ */
+ static Scan createScanForBackupHistory() {
+ Scan scan = new Scan();
+ byte[] startRow = BACKUP_CONTEXT_PREFIX.getBytes();
+ byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
+ stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
+ scan.setStartRow(startRow);
+ scan.setStopRow(stopRow);
+ scan.addFamily(BackupSystemTable.familyName);
+
+ return scan;
+ }
+
+ /**
+ * Converts cell to backup context instance.
+ * @param current - cell
+ * @return backup context instance
+ * @throws IOException exception
+ */
+ static BackupContext cellToBackupContext(Cell current) throws IOException {
+ byte[] data = CellUtil.cloneValue(current);
+ try {
+ BackupContext ctxt = BackupContext.fromByteArray(data);
+ return ctxt;
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Creates Put to write RS last roll log timestamp map
+ * @param table - table
+ * @param smap - map, containing RS:ts
+ * @return put operation
+ */
+ static Put createPutForWriteRegionServerLogTimestamp(String table, String smap) {
+ Put put = new Put((TABLE_RS_LOG_MAP_PREFIX + table).getBytes());
+ put.addColumn(BackupSystemTable.familyName, q0, smap.getBytes());
+ return put;
+ }
+
+ /**
+ * Creates Scan to load table-> { RS -> ts} map of maps
+ * @return scan operation
+ */
+ static Scan createScanForReadLogTimestampMap() {
+ Scan scan = new Scan();
+ byte[] startRow = TABLE_RS_LOG_MAP_PREFIX.getBytes();
+ byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
+ stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
+ scan.setStartRow(startRow);
+ scan.setStopRow(stopRow);
+ scan.addFamily(BackupSystemTable.familyName);
+
+ return scan;
+ }
+
+ /**
+ * Get table name from rowkey
+ * @param cloneRow rowkey
+ * @return table name
+ */
+ static String getTableNameForReadLogTimestampMap(byte[] cloneRow) {
+ int prefixSize = TABLE_RS_LOG_MAP_PREFIX.length();
+ return new String(cloneRow, prefixSize, cloneRow.length - prefixSize);
+ }
+
+ /**
+ * Creates Put to store RS last log result
+ * @param server - server name
+ * @param fileName - log roll result (timestamp)
+ * @return put operation
+ */
+ static Put createPutForRegionServerLastLogRollResult(String server, String fileName) {
+ Put put = new Put((RS_LOG_TS_PREFIX + server).getBytes());
+ put.addColumn(BackupSystemTable.familyName, q0, fileName.getBytes());
+ return put;
+ }
+
+ /**
+ * Creates Scan operation to load last RS log roll results
+ * @return scan operation
+ */
+ static Scan createScanForReadRegionServerLastLogRollResult() {
+ Scan scan = new Scan();
+ byte[] startRow = RS_LOG_TS_PREFIX.getBytes();
+ byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
+ stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
+ scan.setStartRow(startRow);
+ scan.setStopRow(stopRow);
+ scan.addFamily(BackupSystemTable.familyName);
+
+ return scan;
+ }
+
+ /**
+ * Get server's name from rowkey
+ * @param row - rowkey
+ * @return server's name
+ */
+ static String getServerNameForReadRegionServerLastLogRollResult(byte[] row) {
+ int prefixSize = RS_LOG_TS_PREFIX.length();
+ return new String(row, prefixSize, row.length - prefixSize);
+ }
+
+ /**
+ * Creates put list for list of WAL files
+ * @param files list of WAL file paths
+ * @param backupId backup id
+ * @return put list
+ * @throws IOException exception
+ */
+ public static List<Put> createPutsForAddWALFiles(List<String> files, String backupId)
+ throws IOException {
+
+ List<Put> puts = new ArrayList<Put>();
+ for (String file : files) {
+ LOG.debug("+++ put: " + BackupUtil.getUniqueWALFileNamePart(file));
+ byte[] row = (WALS_PREFIX + BackupUtil.getUniqueWALFileNamePart(file)).getBytes();
+ Put put = new Put(row);
+ put.addColumn(BackupSystemTable.familyName, q0, backupId.getBytes());
+ puts.add(put);
+ }
+ return puts;
+ }
+
+ /**
+ * Creates Get operation for a given wal file name
+ * @param file file
+ * @return get operation
+ * @throws IOException exception
+ */
+ public static Get createGetForCheckWALFile(String file) throws IOException {
+ byte[] row = (WALS_PREFIX + BackupUtil.getUniqueWALFileNamePart(file)).getBytes();
+ Get get = new Get(row);
+ get.addFamily(BackupSystemTable.familyName);
+ get.setMaxVersions(1);
+ return get;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java
new file mode 100644
index 0000000..ff8bd2e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java
@@ -0,0 +1,564 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.DefaultWALProvider;
+
+/**
+ * A collection for methods used by multiple classes to backup HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class BackupUtil {
+ protected static final Log LOG = LogFactory.getLog(BackupUtil.class);
+
+ public static final String FIELD_SEPARATOR = "\001";
+ public static final String RECORD_SEPARATOR = "\002";
+ public static final String LOGNAME_SEPARATOR = ".";
+ protected static final String HDFS = "hdfs://";
+ protected static Configuration conf = null;
+
+ private BackupUtil(){
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ /**
+ * Set the configuration from a given one.
+ * @param newConf A new given configuration
+ */
+ public synchronized static void setConf(Configuration newConf) {
+ conf = newConf;
+ }
+
+ /**
+ * Get and merge Hadoop and HBase configuration.
+ * @throws IOException exception
+ */
+ protected static Configuration getConf() {
+ if (conf == null) {
+ conf = new Configuration();
+ HBaseConfiguration.merge(conf, HBaseConfiguration.create());
+ }
+ return conf;
+ }
+
+ /**
+ * Loop through the RS log timestamp map for the tables, for each RS, find the min timestamp value
+ * for the RS among the tables.
+ * @param rsLogTimestampMap timestamp map
+ * @return the min timestamp of each RS
+ */
+ protected static HashMap<String, String> getRSLogTimestampMins(
+ HashMap<String, HashMap<String, String>> rsLogTimestampMap) {
+
+ if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) {
+ return null;
+ }
+
+ HashMap<String, String> rsLogTimestamptMins = new HashMap<String, String>();
+ HashMap<String, HashMap<String, String>> rsLogTimestampMapByRS =
+ new HashMap<String, HashMap<String, String>>();
+
+ for (Entry<String, HashMap<String, String>> tableEntry : rsLogTimestampMap.entrySet()) {
+ String table = tableEntry.getKey();
+ HashMap<String, String> rsLogTimestamp = tableEntry.getValue();
+ for (Entry<String, String> rsEntry : rsLogTimestamp.entrySet()) {
+ String rs = rsEntry.getKey();
+ String ts = rsEntry.getValue();
+ if (!rsLogTimestampMapByRS.containsKey(rs)) {
+ rsLogTimestampMapByRS.put(rs, new HashMap<String, String>());
+ rsLogTimestampMapByRS.get(rs).put(table, ts);
+ } else {
+ rsLogTimestampMapByRS.get(rs).put(table, ts);
+ }
+ }
+ }
+
+ for (String rs : rsLogTimestampMapByRS.keySet()) {
+ rsLogTimestamptMins.put(rs, getMinValue(rsLogTimestampMapByRS.get(rs)));
+ }
+
+ return rsLogTimestamptMins;
+ }
+
+ /**
+ * Get the min value for all the Values a map.
+ * @param map map
+ * @return the min value
+ */
+ protected static String getMinValue(HashMap<String, String> map) {
+ String minTimestamp = null;
+ if (map != null) {
+ ArrayList<String> timestampList = new ArrayList<String>(map.values());
+ Collections.sort(timestampList, new Comparator<String>() {
+ @Override
+ public int compare(String s1, String s2) {
+ long l1 = Long.valueOf(s1);
+ long l2 = Long.valueOf(s2);
+ if (l1 > l2) {
+ return 1;
+ } else if (l1 < l2) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }
+ });
+ // The min among all the RS log timestamps will be kept in ZK.
+ minTimestamp = timestampList.get(0);
+ }
+ return minTimestamp;
+ }
+
+ /**
+ * copy out Table RegionInfo into incremental backup image need to consider move this logic into
+ * HBackupFileSystem
+ * @param backupContext backup context
+ * @param conf configuration
+ * @throws IOException exception
+ * @throws InterruptedException exception
+ */
+ protected static void copyTableRegionInfo(BackupContext backupContext, Configuration conf)
+ throws IOException, InterruptedException {
+
+ Path rootDir = FSUtils.getRootDir(conf);
+ FileSystem fs = rootDir.getFileSystem(conf);
+
+ // for each table in the table set, copy out the table info and region info files in the correct
+ // directory structure
+ for (String table : backupContext.getTables()) {
+
+ LOG.debug("Attempting to copy table info for:" + table);
+ TableDescriptor orig =
+ FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, TableName.valueOf(table));
+
+ // write a copy of descriptor to the target directory
+ Path target = new Path(backupContext.getBackupStatus(table).getTargetDir());
+ FileSystem targetFs = target.getFileSystem(conf);
+ FSTableDescriptors descriptors =
+ new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf));
+ descriptors.createTableDescriptorForTableDirectory(target, orig, false);
+ LOG.debug("Finished copying tableinfo.");
+
+ HBaseAdmin hbadmin = null;
+ // TODO: optimize
+ Connection conn = null;
+ List<HRegionInfo> regions = null;
+ try {
+ conn = ConnectionFactory.createConnection(conf);
+ hbadmin = (HBaseAdmin) conn.getAdmin();
+ regions = hbadmin.getTableRegions(TableName.valueOf(table));
+ } catch (Exception e) {
+ throw new BackupException(e);
+ } finally {
+ if (hbadmin != null) {
+ hbadmin.close();
+ }
+ if(conn != null){
+ conn.close();
+ }
+ }
+
+ // For each region, write the region info to disk
+ LOG.debug("Starting to write region info for table " + table);
+ for (HRegionInfo regionInfo : regions) {
+ Path regionDir =
+ HRegion.getRegionDir(new Path(backupContext.getBackupStatus(table).getTargetDir()),
+ regionInfo);
+ regionDir =
+ new Path(backupContext.getBackupStatus(table).getTargetDir(), regionDir.getName());
+ writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo);
+ }
+ LOG.debug("Finished writing region info for table " + table);
+ }
+ }
+
+ /**
+ * Write the .regioninfo file on-disk.
+ */
+ public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs,
+ final Path regionInfoDir, HRegionInfo regionInfo) throws IOException {
+ final byte[] content = regionInfo.toDelimitedByteArray();
+ Path regionInfoFile = new Path(regionInfoDir, ".regioninfo");
+ // First check to get the permissions
+ FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
+ // Write the RegionInfo file content
+ FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
+ try {
+ out.write(content);
+ } finally {
+ out.close();
+ }
+ }
+
+ /**
+ * TODO: verify the code
+ * @param p path
+ * @return host name
+ * @throws IOException exception
+ */
+ protected static String parseHostFromOldLog(Path p) throws IOException {
+ String n = p.getName();
+ int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
+ String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
+ return ServerName.parseHostname(s);
+ }
+
+ public static String parseHostNameFromLogFile(Path p) throws IOException {
+ if (isArchivedLogFile(p)) {
+ return parseHostFromOldLog(p);
+ } else {
+ return DefaultWALProvider.getServerNameFromWALDirectoryName(p).getHostname();
+ }
+ }
+
+ private static boolean isArchivedLogFile(Path p) {
+ String oldLog = Path.SEPARATOR + HConstants.HREGION_OLDLOGDIR_NAME + Path.SEPARATOR;
+ return p.toString().contains(oldLog);
+ }
+
+ /**
+ * Return WAL file name
+ * @param walFileName WAL file name
+ * @return WAL file name
+ * @throws IOException exception
+ * @throws IllegalArgumentException exception
+ */
+ public static String getUniqueWALFileNamePart(String walFileName) throws IOException {
+ return new Path(walFileName).getName();
+ }
+
+ /**
+ * Return WAL file name
+ * @param p - WAL file path
+ * @return WAL file name
+ * @throws IOException exception
+ */
+ public static String getUniqueWALFileNamePart(Path p) throws IOException {
+ return p.getName();
+ }
+
+ /**
+ * Given the log file, parse the timestamp from the file name. The timestamp is the last number.
+ * @param p a path to the log file
+ * @return the timestamp
+ * @throws IOException exception
+ */
+ protected static String getCreationTime(Path p, Configuration conf) throws IOException {
+ int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);
+ if (idx < 0) {
+ throw new IOException("Cannot parse timestamp from path " + p);
+ }
+ String ts = p.getName().substring(idx + 1);
+ return ts;
+ }
+
+ /**
+ * Get the total length of files under the given directory recursively.
+ * @param fs The hadoop file system
+ * @param dir The target directory
+ * @return the total length of files
+ * @throws IOException exception
+ */
+ public static long getFilesLength(FileSystem fs, Path dir) throws IOException {
+ long totalLength = 0;
+ FileStatus[] files = FSUtils.listStatus(fs, dir);
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ if (fileStatus.isDir()) {
+ totalLength += getFilesLength(fs, fileStatus.getPath());
+ } else {
+ totalLength += fileStatus.getLen();
+ }
+ }
+ }
+ return totalLength;
+ }
+
+ /**
+ * Keep the record for dependency for incremental backup and history info p.s, we may be able to
+ * merge this class into backupImage class later
+ */
+ public static class BackupCompleteData implements Comparable<BackupCompleteData> {
+ private String startTime;
+ private String endTime;
+ private String type;
+ private String backupRootPath;
+ private String tableList;
+ private String backupToken;
+ private String bytesCopied;
+ private List<String> ancestors;
+ private boolean fromExistingSnapshot = false;
+
+ public List<String> getAncestors() {
+ if (fromExistingSnapshot) {
+ return null;
+ }
+ if (this.ancestors == null) {
+ this.ancestors = new ArrayList<String>();
+ }
+ return this.ancestors;
+ }
+
+ public void addAncestor(String backupToken) {
+ this.getAncestors().add(backupToken);
+ }
+
+ public String getBytesCopied() {
+ return bytesCopied;
+ }
+
+ public void setBytesCopied(String bytesCopied) {
+ this.bytesCopied = bytesCopied;
+ }
+
+ public String getBackupToken() {
+ return backupToken;
+ }
+
+ public void setBackupToken(String backupToken) {
+ this.backupToken = backupToken;
+ }
+
+ public String getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(String startTime) {
+ this.startTime = startTime;
+ }
+
+ public String getEndTime() {
+ return endTime;
+ }
+
+ public void setEndTime(String endTime) {
+ this.endTime = endTime;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getBackupRootPath() {
+ return backupRootPath;
+ }
+
+ public void setBackupRootPath(String backupRootPath) {
+ this.backupRootPath = backupRootPath;
+ }
+
+ public String getTableList() {
+ return tableList;
+ }
+
+ public void setTableList(String tableList) {
+ this.tableList = tableList;
+ }
+
+ public boolean fromExistingSnapshot() {
+ return this.fromExistingSnapshot;
+ }
+
+ public void markFromExistingSnapshot() {
+ this.fromExistingSnapshot = true;
+ }
+
+ @Override
+ public int compareTo(BackupCompleteData o) {
+ Long thisTS =
+ new Long(this.getBackupToken().substring(this.getBackupToken().lastIndexOf("_") + 1));
+ Long otherTS =
+ new Long(o.getBackupToken().substring(o.getBackupToken().lastIndexOf("_") + 1));
+ return thisTS.compareTo(otherTS);
+ }
+
+ }
+
+ /**
+ * Sort history list by start time in descending order.
+ * @param historyList history list
+ * @return sorted list of BackupCompleteData
+ */
+ public static ArrayList<BackupCompleteData> sortHistoryListDesc(
+ ArrayList<BackupCompleteData> historyList) {
+ ArrayList<BackupCompleteData> list = new ArrayList<BackupCompleteData>();
+ TreeMap<String, BackupCompleteData> map = new TreeMap<String, BackupCompleteData>();
+ for (BackupCompleteData h : historyList) {
+ map.put(h.getStartTime(), h);
+ }
+ Iterator<String> i = map.descendingKeySet().iterator();
+ while (i.hasNext()) {
+ list.add(map.get(i.next()));
+ }
+ return list;
+ }
+
+ /**
+ * Get list of all WAL files (WALs and archive)
+ * @param c - configuration
+ * @return list of WAL files
+ * @throws IOException exception
+ */
+ public static List<String> getListOfWALFiles(Configuration c) throws IOException {
+ Path rootDir = FSUtils.getRootDir(c);
+ Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+ Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ List<String> logFiles = new ArrayList<String>();
+
+ FileSystem fs = FileSystem.get(c);
+ logFiles = getFiles(fs, logDir, logFiles, null);
+ logFiles = getFiles(fs, oldLogDir, logFiles, null);
+ return logFiles;
+ }
+
+ /**
+ * Get list of all WAL files (WALs and archive)
+ * @param c - configuration
+ * @return list of WAL files
+ * @throws IOException exception
+ */
+ public static List<String> getListOfWALFiles(Configuration c, PathFilter filter)
+ throws IOException {
+ Path rootDir = FSUtils.getRootDir(c);
+ Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+ Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ List<String> logFiles = new ArrayList<String>();
+
+ FileSystem fs = FileSystem.get(c);
+ logFiles = getFiles(fs, logDir, logFiles, filter);
+ logFiles = getFiles(fs, oldLogDir, logFiles, filter);
+ return logFiles;
+ }
+
+ /**
+ * Get list of all old WAL files (WALs and archive)
+ * @param c - configuration
+ * @return list of WAL files
+ * @throws IOException exception
+ */
+ public static List<String> getWALFilesOlderThan(final Configuration c,
+ final HashMap<String, String> hostTimestampMap) throws IOException {
+ Path rootDir = FSUtils.getRootDir(c);
+ Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+ Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ List<String> logFiles = new ArrayList<String>();
+
+ PathFilter filter = new PathFilter() {
+
+ @Override
+ public boolean accept(Path p) {
+ try {
+ if (DefaultWALProvider.isMetaFile(p)) {
+ return false;
+ }
+ String host = BackupUtil.parseHostNameFromLogFile(p);
+ String oldTimestamp = hostTimestampMap.get(host);
+ String currentLogTS = getCreationTime(p, c);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("path=" + p);
+ LOG.debug("oldTimestamp=" + oldTimestamp);
+ LOG.debug("currentLogTS=" + currentLogTS);
+ }
+ return Long.parseLong(currentLogTS) <= Long.parseLong(oldTimestamp);
+ } catch (IOException e) {
+ LOG.error(e);
+ return false;
+ }
+ }
+ };
+ FileSystem fs = FileSystem.get(c);
+ logFiles = getFiles(fs, logDir, logFiles, filter);
+ logFiles = getFiles(fs, oldLogDir, logFiles, filter);
+ return logFiles;
+ }
+
+ private static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files,
+ PathFilter filter) throws FileNotFoundException, IOException {
+
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true);
+
+ while (it.hasNext()) {
+ LocatedFileStatus lfs = it.next();
+ if (lfs.isDirectory()) {
+ continue;
+ }
+ // apply filter
+ if (filter.accept(lfs.getPath())) {
+ files.add(lfs.getPath().toString());
+ LOG.info(lfs.getPath());
+ }
+ }
+ return files;
+ }
+
+ public static String concat(Collection<String> col, String separator) {
+ if (col.size() == 0) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (String s : col) {
+ sb.append(s + separator);
+ }
+ sb.deleteCharAt(sb.lastIndexOf(";"));
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java
new file mode 100644
index 0000000..74411da
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java
@@ -0,0 +1,511 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * View to an on-disk Backup Image FileSytem
+ * Provides the set of methods necessary to interact with the on-disk Backup Image data.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class HBackupFileSystem {
+ public static final Log LOG = LogFactory.getLog(HBackupFileSystem.class);
+
+ private final String RESTORE_TMP_PATH = "/tmp/restoreTemp";
+ private final String[] ignoreDirs = { "recovered.edits" };
+
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path backupRootPath;
+ private final String backupId;
+
+ /**
+ * Create a view to the on-disk Backup Image.
+ * @param conf to use
+ * @param backupPath to where the backup Image stored
+ * @param backupId represent backup Image
+ */
+ HBackupFileSystem(final Configuration conf, final Path backupRootPath, final String backupId)
+ throws IOException {
+ this.conf = conf;
+ this.fs = backupRootPath.getFileSystem(conf);
+ this.backupRootPath = backupRootPath;
+ this.backupId = backupId; // the backup ID for the lead backup Image
+ }
+
+ /**
+ * @param tableName is the table backuped
+ * @return {@link HTableDescriptor} saved in backup image of the table
+ */
+ protected HTableDescriptor getTableDesc(String tableName) throws FileNotFoundException,
+ IOException {
+
+ Path tableInfoPath = this.getTableInfoPath(tableName);
+ LOG.debug("tableInfoPath = " + tableInfoPath.toString());
+ SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
+ LOG.debug("desc = " + desc.getName());
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
+ HTableDescriptor tableDescriptor = manifest.getTableDescriptor();
+ /*
+ * for HBase 0.96 or 0.98 HTableDescriptor tableDescriptor =
+ * FSTableDescriptors.getTableDescriptorFromFs(fs, tableInfoPath);
+ */
+ if (!tableDescriptor.getNameAsString().equals(tableName)) {
+ LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
+ + tableInfoPath.toString());
+ LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString());
+ }
+ return tableDescriptor;
+ }
+
+ /**
+ * Given the backup root dir, backup id and the table name, return the backup image location,
+ * which is also where the backup manifest file is. return value look like:
+ * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/default/t1_dn/backup_1396650096738"
+ * @param backupRootDir backup root directory
+ * @param backupId backup id
+ * @param table table name
+ * @return backupPath String for the particular table
+ */
+ protected static String getTableBackupDir(String backupRootDir, String backupId, String table) {
+ TableName tableName = TableName.valueOf(table);
+ return backupRootDir + File.separator + tableName.getNamespaceAsString() + File.separator
+ + tableName.getQualifierAsString() + File.separator + backupId;
+ }
+
+ /**
+ * Given the backup root dir, backup id and the table name, return the backup image location,
+ * which is also where the backup manifest file is. return value look like:
+ * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/default/t1_dn/backup_1396650096738"
+ * @param tableN table name
+ * @return backupPath for the particular table
+ */
+ protected Path getTableBackupPath(String tableN) {
+ TableName tableName = TableName.valueOf(tableN);
+ return new Path(this.backupRootPath, tableName.getNamespaceAsString() + File.separator
+ + tableName.getQualifierAsString() + File.separator + backupId);
+ }
+
+ /**
+ * return value represent path for:
+ * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/.hbase-snapshot"
+ * @param tableName table name
+ * @return path for snapshot
+ */
+ protected Path getTableSnapshotPath(String tableName) {
+ return new Path(this.getTableBackupPath(tableName), HConstants.SNAPSHOT_DIR_NAME);
+ }
+
+ /**
+ * return value represent path for:
+ * "..../default/t1_dn/backup_1396650096738/.hbase-snapshot/snapshot_1396650097621_default_t1_dn"
+ * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo,
+ * .data.manifest (trunk)
+ * @param tableName table name
+ * @return path to table info
+ * @throws FileNotFoundException exception
+ * @throws IOException exception
+ */
+ protected Path getTableInfoPath(String tableName) throws FileNotFoundException, IOException {
+
+ Path tableSnapShotPath = this.getTableSnapshotPath(tableName);
+ Path tableInfoPath = null;
+
+ // can't build the path directly as the timestamp values are different
+ FileStatus[] snapshots = fs.listStatus(tableSnapShotPath);
+ for (FileStatus snapshot : snapshots) {
+ tableInfoPath = snapshot.getPath();
+ // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest";
+ if (tableInfoPath.getName().endsWith("data.manifest")) {
+ LOG.debug("find Snapshot Manifest");
+ break;
+ }
+ }
+ return tableInfoPath;
+ }
+
+ /**
+ * return value represent path for:
+ * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn"
+ * @param tabelName table name
+ * @return path to table archive
+ * @throws IOException exception
+ */
+ protected Path getTableArchivePath(String tableName) throws IOException {
+ Path baseDir = new Path(getTableBackupPath(tableName), HConstants.HFILE_ARCHIVE_DIRECTORY);
+ Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR);
+ Path archivePath = new Path(dataDir, TableName.valueOf(tableName).getNamespaceAsString());
+ Path tableArchivePath =
+ new Path(archivePath, TableName.valueOf(tableName).getQualifierAsString());
+ if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) {
+ LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists");
+ tableArchivePath = null; // empty table has no archive
+ }
+ return tableArchivePath;
+ }
+
+ /**
+ * Given the backup root dir and the backup id, return the log file location for an incremental
+ * backup.
+ * @param backupRootDir backup root directory
+ * @param backupId backup id
+ * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738"
+ */
+ protected static String getLogBackupDir(String backupRootDir, String backupId) {
+ return backupRootDir + File.separator + HConstants.HREGION_LOGDIR_NAME + File.separator
+ + backupId;
+ }
+
+ protected static Path getLogBackupPath(String backupRootDir, String backupId) {
+ return new Path(getLogBackupDir(backupRootDir, backupId));
+ }
+
+ private Path getManifestPath(String tableName) throws IOException {
+ Path manifestPath = new Path(getTableBackupPath(tableName), BackupManifest.FILE_NAME);
+
+ LOG.debug("Looking for " + manifestPath.toString());
+ if (!fs.exists(manifestPath)) {
+ // check log dir for incremental backup case
+ manifestPath =
+ new Path(getLogBackupDir(this.backupRootPath.toString(), this.backupId) + File.separator
+ + BackupManifest.FILE_NAME);
+ LOG.debug("Looking for " + manifestPath.toString());
+ if (!fs.exists(manifestPath)) {
+ String errorMsg =
+ "Could not find backup manifest for " + backupId + " in " + backupRootPath.toString();
+ throw new IOException(errorMsg);
+ }
+ }
+ return manifestPath;
+ }
+
+ protected BackupManifest getManifest(String tableName) throws IOException {
+ BackupManifest manifest = new BackupManifest(conf, this.getManifestPath(tableName));
+ return manifest;
+ }
+
+ /**
+ * Gets region list
+ * @param tableName table name
+ * @return RegionList region list
+ * @throws FileNotFoundException exception
+ * @throws IOException exception
+ */
+
+ protected ArrayList<Path> getRegionList(String tableName) throws FileNotFoundException,
+ IOException {
+ Path tableArchivePath = this.getTableArchivePath(tableName);
+ ArrayList<Path> regionDirList = new ArrayList<Path>();
+ FileStatus[] children = fs.listStatus(tableArchivePath);
+ for (FileStatus childStatus : children) {
+ // here child refer to each region(Name)
+ Path child = childStatus.getPath();
+ regionDirList.add(child);
+ }
+ return regionDirList;
+ }
+
+ /**
+ * Gets region list
+ * @param tableArchivePath table archive path
+ * @return RegionList region list
+ * @throws FileNotFoundException exception
+ * @throws IOException exception
+ */
+ protected ArrayList<Path> getRegionList(Path tableArchivePath) throws FileNotFoundException,
+ IOException {
+ ArrayList<Path> regionDirList = new ArrayList<Path>();
+ FileStatus[] children = fs.listStatus(tableArchivePath);
+ for (FileStatus childStatus : children) {
+ // here child refer to each region(Name)
+ Path child = childStatus.getPath();
+ regionDirList.add(child);
+ }
+ return regionDirList;
+ }
+
+ /**
+ * Counts the number of files in all subdirectories of an HBase tables, i.e. HFiles. And finds the
+ * maximum number of files in one HBase table.
+ * @param tableArchivePath archive path
+ * @return the maximum number of files found in 1 HBase table
+ * @throws IOException exception
+ */
+ protected int getMaxNumberOfFilesInSubDir(Path tableArchivePath) throws IOException {
+ int result = 1;
+ ArrayList<Path> regionPathList = this.getRegionList(tableArchivePath);
+ // tableArchivePath = this.getTableArchivePath(tableName);
+
+ if (regionPathList == null || regionPathList.size() == 0) {
+ throw new IllegalStateException("Cannot restore hbase table because directory '"
+ + tableArchivePath + "' is not a directory.");
+ }
+
+ for (Path regionPath : regionPathList) {
+ result = Math.max(result, getNumberOfFilesInDir(regionPath));
+ }
+ return result;
+ }
+
+ /**
+ * Counts the number of files in all subdirectories of an HBase table, i.e. HFiles.
+ * @param regionPath Path to an HBase table directory
+ * @return the number of files all directories
+ * @throws IOException exception
+ */
+ protected int getNumberOfFilesInDir(Path regionPath) throws IOException {
+ int result = 0;
+
+ if (!fs.exists(regionPath) || !fs.getFileStatus(regionPath).isDirectory()) {
+ throw new IllegalStateException("Cannot restore hbase table because directory '"
+ + regionPath.toString() + "' is not a directory.");
+ }
+
+ FileStatus[] tableDirContent = fs.listStatus(regionPath);
+ for (FileStatus subDirStatus : tableDirContent) {
+ FileStatus[] colFamilies = fs.listStatus(subDirStatus.getPath());
+ for (FileStatus colFamilyStatus : colFamilies) {
+ FileStatus[] colFamilyContent = fs.listStatus(colFamilyStatus.getPath());
+ result += colFamilyContent.length;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Duplicate the backup image if it's on local cluster
+ * @see HStore#bulkLoadHFile(String, long)
+ * @see HRegionFileSystem#bulkLoadStoreFile(String familyName, Path srcPath, long seqNum)
+ * @param tableArchivePath archive path
+ * @return the new tableArchivePath
+ * @throws IOException exception
+ */
+ protected Path checkLocalAndBackup(Path tableArchivePath) throws IOException {
+ // Move the file if it's on local cluster
+ boolean isCopyNeeded = false;
+
+ FileSystem srcFs = tableArchivePath.getFileSystem(conf);
+ FileSystem desFs = FileSystem.get(conf);
+ if (tableArchivePath.getName().startsWith("/")) {
+ isCopyNeeded = true;
+ } else {
+ // This should match what is done in @see HRegionFileSystem#bulkLoadStoreFile(String, Path,
+ // long)
+ if (srcFs.getUri().equals(desFs.getUri())) {
+ LOG.debug("cluster hold the backup image: " + srcFs.getUri() + "; local cluster node: "
+ + desFs.getUri());
+ isCopyNeeded = true;
+ }
+ }
+ if (isCopyNeeded) {
+ LOG.debug("File " + tableArchivePath + " on local cluster, back it up before restore");
+ Path tmpPath = new Path(RESTORE_TMP_PATH);
+ if (desFs.exists(tmpPath)) {
+ try {
+ desFs.delete(tmpPath, true);
+ } catch (IOException e) {
+ LOG.debug("Failed to delete path: " + tmpPath
+ + ", need to check whether restore target DFS cluster is healthy");
+ }
+ }
+ FileUtil.copy(srcFs, tableArchivePath, desFs, tmpPath, false, conf);
+ LOG.debug("Copied to temporary path on local cluster: " + tmpPath);
+ tableArchivePath = tmpPath;
+ }
+ return tableArchivePath;
+ }
+
+ /**
+ * Calculate region boundaries and add all the column families to the table descriptor
+ * @param regionDirList region dir list
+ * @return a set of keys to store the boundaries
+ */
+ protected byte[][] generateBoundaryKeys(ArrayList<Path> regionDirList)
+ throws FileNotFoundException, IOException {
+ TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
+ // Build a set of keys to store the boundaries
+ byte[][] keys = null;
+ // calculate region boundaries and add all the column families to the table descriptor
+ for (Path regionDir : regionDirList) {
+ LOG.debug("Parsing region dir: " + regionDir);
+ Path hfofDir = regionDir;
+
+ if (!fs.exists(hfofDir)) {
+ LOG.warn("HFileOutputFormat dir " + hfofDir + " not found");
+ }
+
+ FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
+ if (familyDirStatuses == null) {
+ throw new IOException("No families found in " + hfofDir);
+ }
+
+ for (FileStatus stat : familyDirStatuses) {
+ if (!stat.isDirectory()) {
+ LOG.warn("Skipping non-directory " + stat.getPath());
+ continue;
+ }
+ boolean isIgnore = false;
+ String pathName = stat.getPath().getName();
+ for (String ignore : ignoreDirs) {
+ if (pathName.contains(ignore)) {
+ LOG.warn("Skipping non-family directory" + pathName);
+ isIgnore = true;
+ break;
+ }
+ }
+ if (isIgnore) {
+ continue;
+ }
+ Path familyDir = stat.getPath();
+ LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]");
+ // Skip _logs, etc
+ if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) {
+ continue;
+ }
+
+ // start to parse hfile inside one family dir
+ Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
+ for (Path hfile : hfiles) {
+ if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".")
+ || StoreFileInfo.isReference(hfile.getName())
+ || HFileLink.isHFileLink(hfile.getName())) {
+ continue;
+ }
+ HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(conf), conf);
+ final byte[] first, last;
+ try {
+ reader.loadFileInfo();
+ first = reader.getFirstRowKey();
+ last = reader.getLastRowKey();
+ LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first="
+ + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last));
+
+ // To eventually infer start key-end key boundaries
+ Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0;
+ map.put(first, value + 1);
+ value = map.containsKey(last) ? (Integer) map.get(last) : 0;
+ map.put(last, value - 1);
+ } finally {
+ reader.close();
+ }
+ }
+ }
+ }
+ keys = LoadIncrementalHFiles.inferBoundaries(map);
+ return keys;
+ }
+
+ /**
+ * Check whether the backup path exist
+ * @param backupStr backup
+ * @param conf configuration
+ * @return Yes if path exists
+ * @throws IOException exception
+ */
+ protected static boolean checkPathExist(String backupStr, Configuration conf)
+ throws IOException {
+ boolean isExist = false;
+ Path backupPath = new Path(backupStr);
+ FileSystem fileSys = backupPath.getFileSystem(conf);
+ String targetFsScheme = fileSys.getUri().getScheme();
+ LOG.debug("Schema of given url: " + backupStr + " is: " + targetFsScheme);
+ if (fileSys.exists(backupPath)) {
+ isExist = true;
+ }
+ return isExist;
+ }
+
+ /**
+ * Check whether the backup image path and there is manifest file in the path.
+ * @param backupManifestMap If all the manifests are found, then they are put into this map
+ * @param tableArray the tables involved
+ * @throws IOException exception
+ */
+ protected void checkImageManifestExist(HashMap<String, BackupManifest> backupManifestMap,
+ String[] tableArray) throws IOException {
+
+ try {
+ for (String tableName : tableArray) {
+ BackupManifest manifest = this.getManifest(tableName);
+ backupManifestMap.put(tableName, manifest);
+ }
+ } catch (IOException e) {
+ String expMsg = e.getMessage();
+ if (expMsg.contains("No FileSystem for scheme")) {
+ if (expMsg.contains("gpfs")) {
+ LOG.error("Please change to use webhdfs url when "
+ + "the backup image to restore locates on gpfs cluster");
+ } else {
+ LOG.error("Unsupported filesystem scheme found in the backup target url, "
+ + "please check the url to make sure no typo in it");
+ }
+ throw e;
+ } else if (expMsg.contains("no authority supported")) {
+ LOG.error("Please change to use webhdfs url when "
+ + "the backup image to restore locates on gpfs cluster");
+ throw e;
+ } else {
+ LOG.error(expMsg);
+ throw e;
+ }
+ }
+ }
+
+ public static String join(String[] names) {
+ StringBuilder sb = new StringBuilder();
+ String sep = BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND;
+ for (String s : names) {
+ sb.append(sep).append(s);
+ }
+ return sb.toString();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java
new file mode 100644
index 0000000..e91857f
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.DefaultWALProvider;
+
+/**
+ * After a full backup was created, the incremental backup will only store the changes made
+ * after the last full or incremental backup.
+ *
+ * Creating the backup copies the logfiles in .logs and .oldlogs since the last backup timestamp.
+ *
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class IncrementalBackupManager {
+ // parent manager
+ private BackupManager backupManager;
+
+ public static final Log LOG = LogFactory.getLog(IncrementalBackupManager.class);
+
+ public IncrementalBackupManager(BackupManager bm) {
+ this.backupManager = bm;
+ }
+
+ /**
+ * Obtain the list of logs that need to be copied out for this incremental backup. The list is set
+ * in BackupContext.
+ * @param backupContext backup context
+ * @return The new HashMap of RS log timestamps after the log roll for this incremental backup.
+ * @throws IOException exception
+ */
+ public HashMap<String, String> getIncrBackupLogFileList(BackupContext backupContext)
+ throws IOException {
+ List<String> logList;
+ HashMap<String, String> newTimestamps;
+ HashMap<String, String> previousTimestampMins;
+
+ Configuration conf = BackupUtil.getConf();
+ String savedStartCode = backupManager.readBackupStartCode();
+
+ // key: tableName
+ // value: <RegionServer,PreviousTimeStamp>
+ HashMap<String, HashMap<String, String>> previousTimestampMap =
+ backupManager.readLogTimestampMap();
+
+ previousTimestampMins = BackupUtil.getRSLogTimestampMins(previousTimestampMap);
+
+ LOG.debug("StartCode " + savedStartCode + "for backupID " + backupContext.getBackupId());
+ LOG.debug("Timestamps " + previousTimestampMap);
+ // get all new log files from .logs and .oldlogs after last TS and before new timestamp
+ if (savedStartCode == null ||
+ previousTimestampMins == null ||
+ previousTimestampMins.isEmpty()) {
+ throw new IOException("Cannot read any previous back up timestamps from hbase:backup. "
+ + "In order to create an incremental backup, at least one full backup is needed.");
+ }
+
+ HBaseAdmin hbadmin = null;
+ Connection conn = null;
+ try {
+ LOG.info("Execute roll log procedure for incremental backup ...");
+ conn = ConnectionFactory.createConnection(conf);
+ hbadmin = (HBaseAdmin) conn.getAdmin();
+ hbadmin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
+ LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, new HashMap<String, String>());
+ } finally {
+ if (hbadmin != null) {
+ hbadmin.close();
+ }
+ if(conn != null){
+ conn.close();
+ }
+ }
+
+ newTimestamps = backupManager.readRegionServerLastLogRollResult();
+
+ logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf, savedStartCode);
+
+ backupContext.setIncrBackupFileList(logList);
+
+ return newTimestamps;
+ }
+
+ /**
+ * For each region server: get all log files newer than the last timestamps but not newer than the
+ * newest timestamps.
+ * @param olderTimestamps the timestamp for each region server of the last backup.
+ * @param newestTimestamps the timestamp for each region server that the backup should lead to.
+ * @param conf the Hadoop and Hbase configuration
+ * @param savedStartCode the startcode (timestamp) of last successful backup.
+ * @return a list of log files to be backed up
+ * @throws IOException exception
+ */
+ private List<String> getLogFilesForNewBackup(HashMap<String, String> olderTimestamps,
+ HashMap<String, String> newestTimestamps, Configuration conf, String savedStartCode)
+ throws IOException {
+ LOG.debug("In getLogFilesForNewBackup()\n" + "olderTimestamps: " + olderTimestamps
+ + "\n newestTimestamps: " + newestTimestamps);
+ Path rootdir = FSUtils.getRootDir(conf);
+ Path logDir = new Path(rootdir, HConstants.HREGION_LOGDIR_NAME);
+ Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
+ FileSystem fs = rootdir.getFileSystem(conf);
+ NewestLogFilter pathFilter = new NewestLogFilter(conf);
+
+ List<String> resultLogFiles = new ArrayList<String>();
+ List<String> newestLogs = new ArrayList<String>();
+
+ /*
+ * The old region servers and timestamps info we kept in hbase:backup may be out of sync if new
+ * region server is added or existing one lost. We'll deal with it here when processing the
+ * logs. If data in hbase:backup has more hosts, just ignore it. If the .logs directory includes
+ * more hosts, the additional hosts will not have old timestamps to compare with. We'll just use
+ * all the logs in that directory. We always write up-to-date region server and timestamp info
+ * to hbase:backup at the end of successful backup.
+ */
+
+ FileStatus[] rss;
+ Path p;
+ String host;
+ String oldTimeStamp;
+ String currentLogFile;
+ String currentLogTS;
+
+ // Get the files in .logs.
+ rss = fs.listStatus(logDir);
+ for (FileStatus rs : rss) {
+ p = rs.getPath();
+ host = DefaultWALProvider.getServerNameFromWALDirectoryName(p).getHostname();
+ FileStatus[] logs;
+ oldTimeStamp = olderTimestamps.get(host);
+ // It is possible that there is no old timestamp in hbase:backup for this host if
+ // this region server is newly added after our last backup.
+ if (oldTimeStamp == null) {
+ logs = fs.listStatus(p);
+ } else {
+ pathFilter.setLastBackupTS(oldTimeStamp);
+ logs = fs.listStatus(p, pathFilter);
+ }
+ for (FileStatus log : logs) {
+ LOG.debug("currentLogFile: " + log.getPath().toString());
+ if (DefaultWALProvider.isMetaFile(log.getPath())) {
+ LOG.debug("Skip hbase:meta log file: " + log.getPath().getName());
+ continue;
+ }
+ currentLogFile = log.getPath().toString();
+ resultLogFiles.add(currentLogFile);
+ currentLogTS = BackupUtil.getCreationTime(log.getPath(), conf);
+ // newestTimestamps is up-to-date with the current list of hosts
+ // so newestTimestamps.get(host) will not be null.
+ if (Long.valueOf(currentLogTS) > Long.valueOf(newestTimestamps.get(host))) {
+ newestLogs.add(currentLogFile);
+ }
+ }
+ }
+
+ // Include the .oldlogs files too.
+ FileStatus[] oldlogs = fs.listStatus(oldLogDir);
+ for (FileStatus oldlog : oldlogs) {
+ p = oldlog.getPath();
+ currentLogFile = p.toString();
+ if (DefaultWALProvider.isMetaFile(p)) {
+ LOG.debug("Skip .meta log file: " + currentLogFile);
+ continue;
+ }
+ host = BackupUtil.parseHostFromOldLog(p);
+ currentLogTS = BackupUtil.getCreationTime(p, conf);
+ oldTimeStamp = olderTimestamps.get(host);
+ /*
+ * It is possible that there is no old timestamp in hbase:backup for this host. At the time of
+ * our last backup operation, this rs did not exist. The reason can be one of the two: 1. The
+ * rs already left/crashed. Its logs were moved to .oldlogs. 2. The rs was added after our
+ * last backup.
+ */
+ if (oldTimeStamp == null) {
+ if (Long.valueOf(currentLogTS) < Long.valueOf(savedStartCode)) {
+ // This log file is really old, its region server was before our last backup.
+ continue;
+ } else {
+ resultLogFiles.add(currentLogFile);
+ }
+ } else if (Long.valueOf(currentLogTS) > Long.valueOf(oldTimeStamp)) {
+ resultLogFiles.add(currentLogFile);
+ }
+
+ LOG.debug("resultLogFiles before removal of newestLogs: " + resultLogFiles);
+ // It is possible that a host in .oldlogs is an obsolete region server
+ // so newestTimestamps.get(host) here can be null.
+ // Even if these logs belong to a obsolete region server, we still need
+ // to include they to avoid loss of edits for backup.
+ String newTimestamp = newestTimestamps.get(host);
+ if (newTimestamp != null && Long.valueOf(currentLogTS) > Long.valueOf(newTimestamp)) {
+ newestLogs.add(currentLogFile);
+ }
+ }
+ LOG.debug("newestLogs: " + newestLogs);
+ // remove newest log per host because they are still in use
+ resultLogFiles.removeAll(newestLogs);
+ LOG.debug("resultLogFiles after removal of newestLogs: " + resultLogFiles);
+ return resultLogFiles;
+ }
+
+ class NewestLogFilter implements PathFilter {
+ private String lastBackupTS = "0";
+ final private Configuration conf;
+
+ public NewestLogFilter(Configuration conf) {
+ this.conf = conf;
+ }
+
+ protected void setLastBackupTS(String ts) {
+ this.lastBackupTS = ts;
+ }
+
+ @Override
+ public boolean accept(Path path) {
+ // skip meta table log -- ts.meta file
+ if (DefaultWALProvider.isMetaFile(path)) {
+ LOG.debug("Skip .meta log file: " + path.getName());
+ return false;
+ }
+ String timestamp;
+ try {
+ timestamp = BackupUtil.getCreationTime(path, conf);
+ return Long.valueOf(timestamp) > Long.valueOf(lastBackupTS);
+ } catch (IOException e) {
+ LOG.warn("Cannot read timestamp of log file " + path);
+ return false;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java
new file mode 100644
index 0000000..72e4879
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface IncrementalRestoreService extends Configurable{
+
+ public void run(String logDirectory, String[] fromTables, String[] toTables)
+ throws IOException;
+}
[21/29] hbase git commit: HBASE-15018 Inconsistent way of handling
TimeoutException in the rpc client implementations
Posted by sy...@apache.org.
HBASE-15018 Inconsistent way of handling TimeoutException in the rpc client implementations
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/413d663f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/413d663f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/413d663f
Branch: refs/heads/hbase-12439
Commit: 413d663f9262bcdfce67cbc902f7e3153a161fad
Parents: 5e2c2e1
Author: Ashish Singhi <as...@huawei.com>
Authored: Sat Dec 26 22:21:33 2015 +0530
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 29 20:25:47 2015 -0600
----------------------------------------------------------------------
.../hadoop/hbase/ipc/AbstractRpcClient.java | 39 ++++++-
.../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 5 +-
.../apache/hadoop/hbase/ipc/RpcClientImpl.java | 102 +++++++------------
.../hadoop/hbase/ipc/AbstractTestIPC.java | 18 ++++
4 files changed, 92 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/413d663f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
index 6f5e78a..e33ef3a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
@@ -24,6 +24,13 @@ import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.SocketTimeoutException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -34,6 +41,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.codec.KeyValueCodec;
+import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -41,10 +49,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PoolMap;
import org.apache.hadoop.io.compress.CompressionCodec;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-
/**
* Provides the basics for a RpcClient implementation like configuration and Logging.
*/
@@ -258,6 +262,33 @@ public abstract class AbstractRpcClient implements RpcClient {
}
/**
+ * Takes an Exception and the address we were trying to connect to and return an IOException with
+ * the input exception as the cause. The new exception provides the stack trace of the place where
+ * the exception is thrown and some extra diagnostics information. If the exception is
+ * ConnectException or SocketTimeoutException, return a new one of the same type; Otherwise return
+ * an IOException.
+ * @param addr target address
+ * @param exception the relevant exception
+ * @return an exception to throw
+ */
+ protected IOException wrapException(InetSocketAddress addr, Exception exception) {
+ if (exception instanceof ConnectException) {
+ // connection refused; include the host:port in the error
+ return (ConnectException) new ConnectException("Call to " + addr
+ + " failed on connection exception: " + exception).initCause(exception);
+ } else if (exception instanceof SocketTimeoutException) {
+ return (SocketTimeoutException) new SocketTimeoutException("Call to " + addr
+ + " failed because " + exception).initCause(exception);
+ } else if (exception instanceof ConnectionClosingException) {
+ return (ConnectionClosingException) new ConnectionClosingException("Call to " + addr
+ + " failed on local exception: " + exception).initCause(exception);
+ } else {
+ return (IOException) new IOException("Call to " + addr + " failed on local exception: "
+ + exception).initCause(exception);
+ }
+ }
+
+ /**
* Blocking rpc channel that goes via hbase rpc.
*/
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hbase/blob/413d663f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
index f972d0e..c2bd457 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
@@ -251,10 +251,11 @@ public class AsyncRpcClient extends AbstractRpcClient {
if (e.getCause() instanceof IOException) {
throw (IOException) e.getCause();
} else {
- throw new IOException(e.getCause());
+ throw wrapException(addr, (Exception) e.getCause());
}
} catch (TimeoutException e) {
- throw new CallTimeoutException(promise.toString());
+ CallTimeoutException cte = new CallTimeoutException(promise.toString());
+ throw wrapException(addr, cte);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/413d663f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index d7a0029..ca8a149 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -19,11 +19,37 @@
package org.apache.hadoop.hbase.ipc;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Message.Builder;
-import com.google.protobuf.RpcCallback;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.net.SocketFactory;
+import javax.security.sasl.SaslException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -70,37 +96,11 @@ import org.apache.htrace.Span;
import org.apache.htrace.Trace;
import org.apache.htrace.TraceScope;
-import javax.net.SocketFactory;
-import javax.security.sasl.SaslException;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InterruptedIOException;
-import java.io.OutputStream;
-import java.net.ConnectException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketAddress;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Message.Builder;
+import com.google.protobuf.RpcCallback;
/**
* Does RPC against a cluster. Manages connections per regionserver in the cluster.
@@ -1267,36 +1267,6 @@ public class RpcClientImpl extends AbstractRpcClient {
/**
- * Take an IOException and the address we were trying to connect to
- * and return an IOException with the input exception as the cause.
- * The new exception provides the stack trace of the place where
- * the exception is thrown and some extra diagnostics information.
- * If the exception is ConnectException or SocketTimeoutException,
- * return a new one of the same type; Otherwise return an IOException.
- *
- * @param addr target address
- * @param exception the relevant exception
- * @return an exception to throw
- */
- protected IOException wrapException(InetSocketAddress addr,
- IOException exception) {
- if (exception instanceof ConnectException) {
- //connection refused; include the host:port in the error
- return (ConnectException)new ConnectException(
- "Call to " + addr + " failed on connection exception: " + exception).initCause(exception);
- } else if (exception instanceof SocketTimeoutException) {
- return (SocketTimeoutException)new SocketTimeoutException("Call to " + addr +
- " failed because " + exception).initCause(exception);
- } else if (exception instanceof ConnectionClosingException){
- return (ConnectionClosingException) new ConnectionClosingException(
- "Call to " + addr + " failed on local exception: " + exception).initCause(exception);
- } else {
- return (IOException)new IOException("Call to " + addr + " failed on local exception: " +
- exception).initCause(exception);
- }
- }
-
- /**
* Interrupt the connections to the given ip:port server. This should be called if the server
* is known as actually dead. This will not prevent current operation to be retried, and,
* depending on their own behavior, they may retry on the same server. This can be a feature,
http://git-wip-us.apache.org/repos/asf/hbase/blob/413d663f/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
index 5df1edc..ffe4d40 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
@@ -26,8 +26,10 @@ import static org.mockito.Mockito.verify;
import static org.mockito.internal.verification.VerificationModeFactory.times;
import java.io.IOException;
+import java.net.ConnectException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
@@ -42,6 +44,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.MetricsConnection;
+import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoRequestProto;
import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoResponseProto;
import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EmptyRequestProto;
@@ -363,4 +366,19 @@ public abstract class AbstractTestIPC {
rpcServer.stop();
}
}
+
+ @Test
+ public void testWrapException() throws Exception {
+ AbstractRpcClient client =
+ (AbstractRpcClient) RpcClientFactory.createClient(CONF, "AbstractTestIPC");
+ final InetSocketAddress address = InetSocketAddress.createUnresolved("localhost", 0);
+ assertTrue(client.wrapException(address, new ConnectException()) instanceof ConnectException);
+ assertTrue(client.wrapException(address,
+ new SocketTimeoutException()) instanceof SocketTimeoutException);
+ assertTrue(client.wrapException(address, new ConnectionClosingException(
+ "Test AbstractRpcClient#wrapException")) instanceof ConnectionClosingException);
+ assertTrue(client
+ .wrapException(address, new CallTimeoutException("Test AbstractRpcClient#wrapException"))
+ .getCause() instanceof CallTimeoutException);
+ }
}
[03/29] hbase git commit: HBASE-14030 HBase Backup/Restore Phase 1
(Vladimir Rodionov)
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
new file mode 100644
index 0000000..ae21b33
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
@@ -0,0 +1,496 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupManifest.BackupImage;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * The main class which interprets the given arguments and trigger restore operation.
+ */
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class RestoreClient {
+
+ private static final Log LOG = LogFactory.getLog(RestoreClient.class);
+
+ private static Options opt;
+ private static Configuration conf;
+ private static Set<BackupImage> lastRestoreImagesSet;
+
+ // delimiter in tablename list in restore command
+ private static final String DELIMITER_IN_COMMAND = ",";
+
+ private static final String OPTION_OVERWRITE = "overwrite";
+ private static final String OPTION_CHECK = "check";
+ private static final String OPTION_AUTOMATIC = "automatic";
+
+ private static final String USAGE =
+ "Usage: hbase restore <backup_root_path> <backup_id> <tables> [tableMapping] \n"
+ + " [-overwrite] [-check] [-automatic]\n"
+ + " backup_root_path The parent location where the backup images are stored\n"
+ + " backup_id The id identifying the backup image\n"
+ + " table(s) Table(s) from the backup image to be restored.\n"
+ + " Tables are separated by comma.\n"
+ + " Options:\n"
+ + " tableMapping A comma separated list of target tables.\n"
+ + " If specified, each table in <tables> must have a mapping.\n"
+ + " -overwrite With this option, restore overwrites to the existing table "
+ + "if there's any in\n"
+ + " restore target. The existing table must be online before restore.\n"
+ + " -check With this option, restore sequence and dependencies are checked\n"
+ + " and verified without executing the restore\n"
+ + " -automatic With this option, all the dependencies are automatically restored\n"
+ + " together with this backup image following the correct order.\n"
+ + " The restore dependencies can be checked by using \"-check\" "
+ + "option,\n"
+ + " or using \"hbase backup describe\" command. Without this option, "
+ + "only\n" + " this backup image is restored\n";
+
+ private RestoreClient(){
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ protected static void init() throws IOException {
+ // define supported options
+ opt = new Options();
+ opt.addOption(OPTION_OVERWRITE, false,
+ "Overwrite the data if any of the restore target tables exists");
+ opt.addOption(OPTION_CHECK, false, "Check restore sequence and dependencies");
+ opt.addOption(OPTION_AUTOMATIC, false, "Restore all dependencies");
+ opt.addOption("debug", false, "Enable debug logging");
+
+ conf = getConf();
+
+ // disable irrelevant loggers to avoid it mess up command output
+ disableUselessLoggers();
+ }
+
+ public static void main(String[] args) throws IOException {
+ init();
+ parseAndRun(args);
+ }
+
+ private static void parseAndRun(String[] args) {
+ CommandLine cmd = null;
+ try {
+ cmd = new PosixParser().parse(opt, args);
+ } catch (ParseException e) {
+ LOG.error("Could not parse command", e);
+ System.exit(-1);
+ }
+
+ // enable debug logging
+ Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup");
+ if (cmd.hasOption("debug")) {
+ backupClientLogger.setLevel(Level.DEBUG);
+ }
+
+ // whether to overwrite to existing table if any, false by default
+ boolean isOverwrite = cmd.hasOption(OPTION_OVERWRITE);
+ if (isOverwrite) {
+ LOG.debug("Found -overwrite option in restore command, "
+ + "will overwrite to existing table if any in the restore target");
+ }
+
+ // whether to only check the dependencies, false by default
+ boolean check = cmd.hasOption(OPTION_CHECK);
+ if (check) {
+ LOG.debug("Found -check option in restore command, "
+ + "will check and verify the dependencies");
+ }
+
+ // whether to restore all dependencies, false by default
+ boolean autoRestore = cmd.hasOption(OPTION_AUTOMATIC);
+ if (autoRestore) {
+ LOG.debug("Found -automatic option in restore command, "
+ + "will automatically retore all the dependencies");
+ }
+
+ // parse main restore command options
+ String[] remainArgs = cmd.getArgs();
+ if (remainArgs.length < 3) {
+ System.out.println("ERROR: missing arguments");
+ System.out.println(USAGE);
+ System.exit(-1);
+ }
+
+ String backupRootDir = remainArgs[0];
+ String backupId = remainArgs[1];
+ String tables = remainArgs[2];
+
+ String tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null;
+
+ String[] sTableArray = (tables != null) ? tables.split(DELIMITER_IN_COMMAND) : null;
+ String[] tTableArray = (tableMapping != null) ? tableMapping.split(DELIMITER_IN_COMMAND) : null;
+
+ if (tableMapping != null && tTableArray != null && (sTableArray.length != tTableArray.length)) {
+ System.err.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping);
+ System.out.println(USAGE);
+ System.exit(-1);
+ }
+
+ try {
+ HBackupFileSystem hBackupFS = new HBackupFileSystem(conf, new Path(backupRootDir), backupId);
+ restore_stage1(hBackupFS, backupRootDir, backupId, check, autoRestore, sTableArray,
+ tTableArray, isOverwrite);
+ } catch (IOException e) {
+ System.err.println("ERROR: " + e.getMessage());
+ System.exit(-1);
+ }
+ }
+
+ /**
+ * Restore operation. Stage 1: validate backupManifest, and check target tables
+ * @param hBackupFS to access the backup image
+ * @param backupRootDir The root dir for backup image
+ * @param backupId The backup id for image to be restored
+ * @param check True if only do dependency check
+ * @param autoRestore True if automatically restore following the dependency
+ * @param sTableArray The array of tables to be restored
+ * @param tTableArray The array of mapping tables to restore to
+ * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the
+ * request if target table exists
+ * @return True if only do dependency check
+ * @throws IOException if any failure during restore
+ */
+ public static boolean restore_stage1(HBackupFileSystem hBackupFS, String backupRootDir,
+ String backupId, boolean check, boolean autoRestore, String[] sTableArray,
+ String[] tTableArray, boolean isOverwrite) throws IOException {
+
+ HashMap<String, BackupManifest> backupManifestMap = new HashMap<String, BackupManifest>();
+ // check and load backup image manifest for the tables
+ hBackupFS.checkImageManifestExist(backupManifestMap, sTableArray);
+
+ try {
+ // Check and validate the backup image and its dependencies
+ if (check || autoRestore) {
+ if (validate(backupManifestMap)) {
+ LOG.info("Checking backup images: ok");
+ } else {
+ String errMsg = "Some dependencies are missing for restore";
+ LOG.error(errMsg);
+ throw new IOException(errMsg);
+ }
+ }
+
+ // return true if only for check
+ if (check) {
+ return true;
+ }
+
+ if (tTableArray == null) {
+ tTableArray = sTableArray;
+ }
+
+ // check the target tables
+ checkTargetTables(tTableArray, isOverwrite);
+
+ // start restore process
+ Set<BackupImage> restoreImageSet =
+ restore_stage2(hBackupFS, backupManifestMap, sTableArray, tTableArray, autoRestore);
+
+ LOG.info("Restore for " + Arrays.asList(sTableArray) + " are successful!");
+ lastRestoreImagesSet = restoreImageSet;
+
+ } catch (IOException e) {
+ LOG.error("ERROR: restore failed with error: " + e.getMessage());
+ throw e;
+ }
+
+ // not only for check, return false
+ return false;
+ }
+
+ /**
+ * Get last restore image set. The value is globally set for the latest finished restore.
+ * @return the last restore image set
+ */
+ public static Set<BackupImage> getLastRestoreImagesSet() {
+ return lastRestoreImagesSet;
+ }
+
+ private static boolean validate(HashMap<String, BackupManifest> backupManifestMap)
+ throws IOException {
+ boolean isValid = true;
+
+ for (Entry<String, BackupManifest> manifestEntry : backupManifestMap.entrySet()) {
+
+ String table = manifestEntry.getKey();
+ TreeSet<BackupImage> imageSet = new TreeSet<BackupImage>();
+
+ ArrayList<BackupImage> depList = manifestEntry.getValue().getDependentListByTable(table);
+ if (depList != null && !depList.isEmpty()) {
+ imageSet.addAll(depList);
+ }
+
+ // todo merge
+ LOG.debug("merge will be implemented in future jira");
+ // BackupUtil.clearMergedImages(table, imageSet, conf);
+
+ LOG.info("Dependent image(s) from old to new:");
+ for (BackupImage image : imageSet) {
+ String imageDir =
+ HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table);
+ if (!HBackupFileSystem.checkPathExist(imageDir, getConf())) {
+ LOG.error("ERROR: backup image does not exist: " + imageDir);
+ isValid = false;
+ break;
+ }
+ // TODO More validation?
+ LOG.info("Backup image: " + image.getBackupId() + " for '" + table + "' is available");
+ }
+ }
+ return isValid;
+ }
+
+ /**
+ * Validate target Tables
+ * @param tTableArray: target tables
+ * @param isOverwrite overwrite existing table
+ * @throws IOException exception
+ */
+ private static void checkTargetTables(String[] tTableArray, boolean isOverwrite)
+ throws IOException {
+ ArrayList<String> existTableList = new ArrayList<String>();
+ ArrayList<String> disabledTableList = new ArrayList<String>();
+
+ // check if the tables already exist
+ HBaseAdmin admin = null;
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(conf);
+ admin = (HBaseAdmin) conn.getAdmin();
+ for (String tableName : tTableArray) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
+ existTableList.add(tableName);
+ if (admin.isTableDisabled(TableName.valueOf(tableName))) {
+ disabledTableList.add(tableName);
+ }
+ } else {
+ LOG.info("HBase table " + tableName
+ + " does not exist. It will be create during backup process");
+ }
+ }
+ } finally {
+ if (admin != null) {
+ admin.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+
+ if (existTableList.size() > 0) {
+ if (!isOverwrite) {
+ LOG.error("Existing table found in the restore target, please add \"-overwrite\" "
+ + "option in the command if you mean to restore to these existing tables");
+ LOG.info("Existing table list in restore target: " + existTableList);
+ throw new IOException("Existing table found in target while no \"-overwrite\" "
+ + "option found");
+ } else {
+ if (disabledTableList.size() > 0) {
+ LOG.error("Found offline table in the restore target, "
+ + "please enable them before restore with \"-overwrite\" option");
+ LOG.info("Offline table list in restore target: " + disabledTableList);
+ throw new IOException(
+ "Found offline table in the target when restore with \"-overwrite\" option");
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Restore operation. Stage 2: resolved Backup Image dependency
+ * @param hBackupFS to access the backup image
+ * @param backupManifestMap : tableName, Manifest
+ * @param sTableArray The array of tables to be restored
+ * @param tTableArray The array of mapping tables to restore to
+ * @param autoRestore : yes, restore all the backup images on the dependency list
+ * @return set of BackupImages restored
+ * @throws IOException exception
+ */
+ private static Set<BackupImage> restore_stage2(HBackupFileSystem hBackupFS,
+ HashMap<String, BackupManifest> backupManifestMap, String[] sTableArray,
+ String[] tTableArray, boolean autoRestore) throws IOException {
+ TreeSet<BackupImage> restoreImageSet = new TreeSet<BackupImage>();
+
+ for (int i = 0; i < sTableArray.length; i++) {
+ restoreImageSet.clear();
+ String table = sTableArray[i];
+ BackupManifest manifest = backupManifestMap.get(table);
+ if (autoRestore) {
+ // Get the image list of this backup for restore in time order from old
+ // to new.
+ TreeSet<BackupImage> restoreList =
+ new TreeSet<BackupImage>(manifest.getDependentListByTable(table));
+ LOG.debug("need to clear merged Image. to be implemented in future jira");
+
+ for (BackupImage image : restoreList) {
+ restoreImage(image, table, tTableArray[i]);
+ }
+ restoreImageSet.addAll(restoreList);
+ } else {
+ BackupImage image = manifest.getBackupImage();
+ List<BackupImage> depList = manifest.getDependentListByTable(table);
+ // The dependency list always contains self.
+ if (depList != null && depList.size() > 1) {
+ LOG.warn("Backup image " + image.getBackupId() + " depends on other images.\n"
+ + "this operation will only restore the delta contained within backupImage "
+ + image.getBackupId());
+ }
+ restoreImage(image, table, tTableArray[i]);
+ restoreImageSet.add(image);
+ }
+
+ if (autoRestore) {
+ if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
+ LOG.info("Restore includes the following image(s):");
+ for (BackupImage image : restoreImageSet) {
+ LOG.info(" Backup: "
+ + image.getBackupId()
+ + " "
+ + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(),
+ table));
+ }
+ }
+ }
+
+ }
+ return restoreImageSet;
+ }
+
+ /**
+ * Restore operation handle each backupImage
+ * @param image: backupImage
+ * @param sTable: table to be restored
+ * @param tTable: table to be restored to
+ * @throws IOException exception
+ */
+ private static void restoreImage(BackupImage image, String sTable, String tTable)
+ throws IOException {
+
+ Configuration conf = getConf();
+
+ String rootDir = image.getRootDir();
+ LOG.debug("Image root dir " + rootDir);
+ String backupId = image.getBackupId();
+
+ HBackupFileSystem hFS = new HBackupFileSystem(conf, new Path(rootDir), backupId);
+ RestoreUtil restoreTool = new RestoreUtil(conf, hFS);
+ BackupManifest manifest = hFS.getManifest(sTable);
+
+ Path tableBackupPath = hFS.getTableBackupPath(sTable);
+
+ // todo: convert feature will be provided in a future jira
+ boolean converted = false;
+
+ if (manifest.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL) || converted) {
+ LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from "
+ + (converted ? "converted" : "full") + " backup image " + tableBackupPath.toString());
+ restoreTool.fullRestoreTable(tableBackupPath, sTable, tTable, converted);
+ } else { // incremental Backup
+ String logBackupDir =
+ HBackupFileSystem.getLogBackupDir(image.getRootDir(), image.getBackupId());
+ LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from incremental backup image "
+ + logBackupDir);
+ restoreTool.incrementalRestoreTable(logBackupDir, new String[] { sTable },
+ new String[] { tTable });
+ }
+
+ LOG.info(sTable + " has been successfully restored to " + tTable);
+ }
+
+ /**
+ * Set the configuration from a given one.
+ * @param newConf A new given configuration
+ */
+ public synchronized static void setConf(Configuration newConf) {
+ conf = newConf;
+ }
+
+ /**
+ * Get and merge Hadoop and HBase configuration.
+ * @throws IOException exception
+ */
+ protected static Configuration getConf() {
+ if (conf == null) {
+ synchronized (RestoreClient.class) {
+ conf = new Configuration();
+ HBaseConfiguration.merge(conf, HBaseConfiguration.create());
+ }
+ }
+ return conf;
+ }
+
+ private static void disableUselessLoggers() {
+ // disable zookeeper log to avoid it mess up command output
+ Logger zkLogger = Logger.getLogger("org.apache.zookeeper");
+ LOG.debug("Zookeeper log level before set: " + zkLogger.getLevel());
+ zkLogger.setLevel(Level.OFF);
+ LOG.debug("Zookeeper log level after set: " + zkLogger.getLevel());
+
+ // disable hbase zookeeper tool log to avoid it mess up command output
+ Logger hbaseZkLogger = Logger.getLogger("org.apache.hadoop.hbase.zookeeper");
+ LOG.debug("HBase zookeeper log level before set: " + hbaseZkLogger.getLevel());
+ hbaseZkLogger.setLevel(Level.OFF);
+ LOG.debug("HBase Zookeeper log level after set: " + hbaseZkLogger.getLevel());
+
+ // disable hbase client log to avoid it mess up command output
+ Logger hbaseClientLogger = Logger.getLogger("org.apache.hadoop.hbase.client");
+ LOG.debug("HBase client log level before set: " + hbaseClientLogger.getLevel());
+ hbaseClientLogger.setLevel(Level.OFF);
+ LOG.debug("HBase client log level after set: " + hbaseClientLogger.getLevel());
+
+ // disable other related log to avoid mess up command output
+ Logger otherLogger = Logger.getLogger("org.apache.hadoop.hbase.io.hfile");
+ otherLogger.setLevel(Level.OFF);
+ otherLogger = Logger.getLogger("org.apache.hadoop.hbase.util");
+ otherLogger.setLevel(Level.OFF);
+ otherLogger = Logger.getLogger("org.apache.hadoop.hbase.mapreduce");
+ otherLogger.setLevel(Level.OFF);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java
new file mode 100644
index 0000000..bdb7988
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreUtil.java
@@ -0,0 +1,503 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.NavigableSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hbase.wal.WALKey;
+import org.apache.hadoop.hbase.wal.WALSplitter;
+
+/**
+ * A collection for methods used by multiple classes to restore HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class RestoreUtil {
+
+ public static final Log LOG = LogFactory.getLog(RestoreUtil.class);
+
+ protected Configuration conf = null;
+
+ protected HBackupFileSystem hBackupFS = null;
+
+ // store table name and snapshot dir mapping
+ private final HashMap<String, Path> snapshotMap = new HashMap<String, Path>();
+
+ public RestoreUtil(Configuration conf, HBackupFileSystem hBackupFS) throws IOException {
+ this.conf = conf;
+ this.hBackupFS = hBackupFS;
+ }
+
+ /**
+ * During incremental backup operation. Call WalPlayer to replay WAL in backup image Currently
+ * tableNames and newTablesNames only contain single table, will be expanded to multiple tables in
+ * the future
+ * @param logDir : incremental backup folders, which contains WAL
+ * @param tableNames : source tableNames(table names were backuped)
+ * @param newTableNames : target tableNames(table names to be restored to)
+ * @throws IOException exception
+ */
+ public void incrementalRestoreTable(String logDir, String[] tableNames, String[] newTableNames)
+ throws IOException {
+
+ if (tableNames.length != newTableNames.length) {
+ throw new IOException("Number of source tables adn taget Tables does not match!");
+ }
+
+ // for incremental backup image, expect the table already created either by user or previous
+ // full backup. Here, check that all new tables exists
+ HBaseAdmin admin = null;
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(conf);
+ admin = (HBaseAdmin) conn.getAdmin();
+ for (String tableName : newTableNames) {
+ if (!admin.tableExists(TableName.valueOf(tableName))) {
+ admin.close();
+ throw new IOException("HBase table " + tableName
+ + " does not exist. Create the table first, e.g. by restoring a full backup.");
+ }
+ }
+ IncrementalRestoreService restoreService =
+ BackupRestoreServiceFactory.getIncrementalRestoreService(conf);
+
+ restoreService.run(logDir, tableNames, newTableNames);
+ } finally {
+ if (admin != null) {
+ admin.close();
+ }
+ if(conn != null){
+ conn.close();
+ }
+ }
+ }
+
+ public void fullRestoreTable(Path tableBackupPath, String tableName, String newTableName,
+ boolean converted) throws IOException {
+
+ restoreTableAndCreate(tableName, newTableName, tableBackupPath, converted);
+ }
+
+ private void restoreTableAndCreate(String tableName, String newTableName, Path tableBackupPath,
+ boolean converted) throws IOException {
+ if (newTableName == null || newTableName.equals("")) {
+ newTableName = tableName;
+ }
+
+ FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);
+
+ // get table descriptor first
+ HTableDescriptor tableDescriptor = null;
+
+ Path tableSnapshotPath = hBackupFS.getTableSnapshotPath(tableName);
+
+ if (fileSys.exists(tableSnapshotPath)) {
+ // snapshot path exist means the backup path is in HDFS
+ // check whether snapshot dir already recorded for target table
+ if (snapshotMap.get(tableName) != null) {
+ SnapshotDescription desc =
+ SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
+ tableDescriptor = manifest.getTableDescriptor();
+ LOG.debug("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString()
+ + " while tableName = " + tableName);
+ // HBase 96.0 and 98.0
+ // tableDescriptor =
+ // FSTableDescriptors.getTableDescriptorFromFs(fileSys, snapshotMap.get(tableName));
+ } else {
+ tableDescriptor = hBackupFS.getTableDesc(tableName);
+ LOG.debug("tableSnapshotPath=" + tableSnapshotPath.toString());
+ snapshotMap.put(tableName, hBackupFS.getTableInfoPath(tableName));
+ }
+ if (tableDescriptor == null) {
+ LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
+ }
+ } else if (converted) {
+ // first check if this is a converted backup image
+ LOG.error("convert will be supported in a future jira");
+ }
+
+ Path tableArchivePath = hBackupFS.getTableArchivePath(tableName);
+ if (tableArchivePath == null) {
+ if (tableDescriptor != null) {
+ // find table descriptor but no archive dir means the table is empty, create table and exit
+ LOG.debug("find table descriptor but no archive dir for table " + tableName
+ + ", will only create table");
+ tableDescriptor.setName(Bytes.toBytes(newTableName));
+ checkAndCreateTable(tableBackupPath, tableName, newTableName, null, tableDescriptor);
+ return;
+ } else {
+ throw new IllegalStateException("Cannot restore hbase table because directory '"
+ + " tableArchivePath is null.");
+ }
+ }
+
+ if (tableDescriptor == null) {
+ tableDescriptor = new HTableDescriptor(newTableName);
+ } else {
+ tableDescriptor.setName(Bytes.toBytes(newTableName));
+ }
+
+ if (!converted) {
+ // record all region dirs:
+ // load all files in dir
+ try {
+ ArrayList<Path> regionPathList = hBackupFS.getRegionList(tableName);
+
+ // should only try to create the table with all region informations, so we could pre-split
+ // the regions in fine grain
+ checkAndCreateTable(tableBackupPath, tableName, newTableName, regionPathList,
+ tableDescriptor);
+ if (tableArchivePath != null) {
+ // start real restore through bulkload
+ // if the backup target is on local cluster, special action needed
+ Path tempTableArchivePath = hBackupFS.checkLocalAndBackup(tableArchivePath);
+ if (tempTableArchivePath.equals(tableArchivePath)) {
+ LOG.debug("TableArchivePath for bulkload using existPath: " + tableArchivePath);
+ } else {
+ regionPathList = hBackupFS.getRegionList(tempTableArchivePath); // point to the tempDir
+ LOG.debug("TableArchivePath for bulkload using tempPath: " + tempTableArchivePath);
+ }
+
+ LoadIncrementalHFiles loader = createLoader(tempTableArchivePath, false);
+ for (Path regionPath : regionPathList) {
+ String regionName = regionPath.toString();
+ LOG.debug("Restoring HFiles from directory " + regionName);
+ String[] args = { regionName, newTableName };
+ loader.run(args);
+ }
+ }
+ // restore the recovered.edits if exists
+ replayRecoveredEditsIfAny(tableBackupPath, tableName, tableDescriptor);
+ } catch (Exception e) {
+ throw new IllegalStateException("Cannot restore hbase table", e);
+ }
+ } else {
+ LOG.debug("convert will be supported in a future jira");
+ }
+ }
+
+ /**
+ * Replay recovered edits from backup.
+ */
+ private void replayRecoveredEditsIfAny(Path tableBackupPath, String tableName,
+ HTableDescriptor newTableHtd) throws IOException {
+
+ LOG.debug("Trying to replay the recovered.edits if exist to the target table "
+ + newTableHtd.getNameAsString() + " from the backup of table " + tableName + ".");
+
+ FileSystem fs = tableBackupPath.getFileSystem(this.conf);
+ ArrayList<Path> regionDirs = hBackupFS.getRegionList(tableName);
+
+ if (regionDirs == null || regionDirs.size() == 0) {
+ LOG.warn("No recovered.edits to be replayed for empty backup of table " + tableName + ".");
+ return;
+ }
+
+ Connection conn = null;
+ try {
+
+ conn = ConnectionFactory.createConnection(conf);
+
+ for (Path regionDir : regionDirs) {
+ // OLD: NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regionDir);
+ NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regionDir);
+
+ if (files == null || files.isEmpty()) {
+ LOG.warn("No recovered.edits found for the region " + regionDir.getName() + ".");
+ return;
+ }
+
+ for (Path edits : files) {
+ if (edits == null || !fs.exists(edits)) {
+ LOG.warn("Null or non-existent edits file: " + edits);
+ continue;
+ }
+
+ HTable table = null;
+ try {
+ table = (HTable) conn.getTable(newTableHtd.getTableName());
+ replayRecoveredEdits(table, fs, edits);
+ table.flushCommits();
+ table.close();
+ } catch (IOException e) {
+ boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
+ if (skipErrors) {
+ Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
+ LOG.error(HConstants.HREGION_EDITS_REPLAY_SKIP_ERRORS
+ + "=true so continuing. Renamed " + edits + " as " + p, e);
+ } else {
+ throw e;
+ }
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ } // for each edit file under a region
+ } // for each region
+
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ /**
+ * Restore process for an edit entry.
+ * @param htable The target table of restore
+ * @param key HLog key
+ * @param val KVs
+ * @throws IOException exception
+ */
+ private void restoreEdit(HTable htable, WALKey key, WALEdit val) throws IOException {
+ Put put = null;
+ Delete del = null;
+ Cell lastKV = null;
+ for (Cell kv : val.getCells()) {
+ // filtering HLog meta entries, see HLog.completeCacheFlushLogEdit
+ if (WALEdit.isMetaEditFamily(CellUtil.cloneFamily(kv))) {
+ continue;
+ }
+
+ // A WALEdit may contain multiple operations (HBASE-3584) and/or
+ // multiple rows (HBASE-5229).
+ // Aggregate as much as possible into a single Put/Delete
+ // operation before apply the action to the table.
+ if (lastKV == null || lastKV.getTypeByte() != kv.getTypeByte()
+ || !CellUtil.matchingRow(lastKV, kv)) {
+ // row or type changed, write out aggregate KVs.
+ if (put != null) {
+ applyAction(htable, put);
+ }
+ if (del != null) {
+ applyAction(htable, del);
+ }
+
+ if (CellUtil.isDelete(kv)) {
+ del = new Delete(CellUtil.cloneRow(kv));
+ } else {
+ put = new Put(CellUtil.cloneRow(kv));
+ }
+ }
+ if (CellUtil.isDelete(kv)) {
+ del.addDeleteMarker(kv);
+ } else {
+ put.add(kv);
+ }
+ lastKV = kv;
+ }
+ // write residual KVs
+ if (put != null) {
+ applyAction(htable, put);
+ }
+ if (del != null) {
+ applyAction(htable, del);
+ }
+ }
+
+ /**
+ * Apply an action (Put/Delete) to table.
+ * @param table table
+ * @param action action
+ * @throws IOException exception
+ */
+ private void applyAction(HTable table, Mutation action) throws IOException {
+ // The actions are not immutable, so we defensively copy them
+ if (action instanceof Put) {
+ Put put = new Put((Put) action);
+ // put.setWriteToWAL(false);
+ // why do not we do WAL?
+ put.setDurability(Durability.SKIP_WAL);
+ table.put(put);
+ } else if (action instanceof Delete) {
+ Delete delete = new Delete((Delete) action);
+ table.delete(delete);
+ } else {
+ throw new IllegalArgumentException("action must be either Delete or Put");
+ }
+ }
+
+ /**
+ * Replay the given edits.
+ * @param htable The target table of restore
+ * @param fs File system
+ * @param edits Recovered.edits to be replayed
+ * @throws IOException exception
+ */
+ private void replayRecoveredEdits(HTable htable, FileSystem fs, Path edits) throws IOException {
+ LOG.debug("Replaying edits from " + edits + "; path=" + edits);
+
+ WAL.Reader reader = null;
+ try {
+ reader = WALFactory.createReader(fs, edits, this.conf);
+ long editsCount = 0;
+ WAL.Entry entry;
+
+ try {
+ while ((entry = reader.next()) != null) {
+ restoreEdit(htable, entry.getKey(), entry.getEdit());
+ editsCount++;
+ }
+ LOG.debug(editsCount + " edits from " + edits + " have been replayed.");
+
+ } catch (EOFException eof) {
+ Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
+ String msg =
+ "Encountered EOF. Most likely due to Master failure during "
+ + "log spliting, so we have this data in another edit. "
+ + "Continuing, but renaming " + edits + " as " + p;
+ LOG.warn(msg, eof);
+ } catch (IOException ioe) {
+ // If the IOE resulted from bad file format,
+ // then this problem is idempotent and retrying won't help
+ if (ioe.getCause() instanceof ParseException) {
+ Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
+ String msg =
+ "File corruption encountered! " + "Continuing, but renaming " + edits + " as " + p;
+ LOG.warn(msg, ioe);
+ } else {
+ // other IO errors may be transient (bad network connection,
+ // checksum exception on one datanode, etc). throw & retry
+ throw ioe;
+ }
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ }
+
+ /**
+ * Create a {@link LoadIncrementalHFiles} instance to be used to restore the HFiles of a full
+ * backup.
+ * @return the {@link LoadIncrementalHFiles} instance
+ * @throws IOException exception
+ */
+ private LoadIncrementalHFiles createLoader(Path tableArchivePath, boolean multipleTables)
+ throws IOException {
+ // set configuration for restore:
+ // LoadIncrementalHFile needs more time
+ // <name>hbase.rpc.timeout</name> <value>600000</value>
+ // calculates
+ Integer milliSecInMin = 60000;
+ Integer previousMillis = this.conf.getInt("hbase.rpc.timeout", 0);
+ Integer numberOfFilesInDir =
+ multipleTables ? hBackupFS.getMaxNumberOfFilesInSubDir(tableArchivePath) : hBackupFS
+ .getNumberOfFilesInDir(tableArchivePath);
+ Integer calculatedMillis = numberOfFilesInDir * milliSecInMin; // 1 minute per file
+ Integer resultMillis = Math.max(calculatedMillis, previousMillis);
+ if (resultMillis > previousMillis) {
+ LOG.info("Setting configuration for restore with LoadIncrementalHFile: "
+ + "hbase.rpc.timeout to " + calculatedMillis / milliSecInMin
+ + " minutes, to handle the number of files in backup " + tableArchivePath);
+ this.conf.setInt("hbase.rpc.timeout", resultMillis);
+ }
+
+ LoadIncrementalHFiles loader = null;
+ try {
+ loader = new LoadIncrementalHFiles(this.conf);
+ } catch (Exception e1) {
+ throw new IOException(e1);
+ }
+ return loader;
+ }
+
+ /**
+ * Prepare the table for bulkload, most codes copied from
+ * {@link LoadIncrementalHFiles#createTable(String, String)}
+ * @param tableBackupPath path
+ * @param tableName table name
+ * @param targetTableName target table name
+ * @param regionDirList region directory list
+ * @param htd table descriptor
+ * @throws IOException exception
+ */
+ private void checkAndCreateTable(Path tableBackupPath, String tableName, String targetTableName,
+ ArrayList<Path> regionDirList, HTableDescriptor htd) throws IOException {
+ HBaseAdmin hbadmin = null;
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(conf);
+ hbadmin = (HBaseAdmin) conn.getAdmin();
+ if (hbadmin.tableExists(TableName.valueOf(targetTableName))) {
+ LOG.info("Using exising target table '" + targetTableName + "'");
+ } else {
+ LOG.info("Creating target table '" + targetTableName + "'");
+
+ // if no region dir given, create the table and return
+ if (regionDirList == null || regionDirList.size() == 0) {
+
+ hbadmin.createTable(htd);
+ return;
+ }
+
+ byte[][] keys = hBackupFS.generateBoundaryKeys(regionDirList);
+
+ // create table using table decriptor and region boundaries
+ hbadmin.createTable(htd, keys);
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ } finally {
+ if (hbadmin != null) {
+ hbadmin.close();
+ }
+ if(conn != null){
+ conn.close();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java
new file mode 100644
index 0000000..a3b5db5
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyService.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.mapreduce;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.backup.BackupCopyService;
+import org.apache.hadoop.hbase.backup.BackupHandler;
+import org.apache.hadoop.hbase.backup.BackupUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpOptions;
+/**
+ * Copier for backup operation. Basically, there are 2 types of copy. One is copying from snapshot,
+ * which bases on extending ExportSnapshot's function with copy progress reporting to ZooKeeper
+ * implementation. The other is copying for incremental log files, which bases on extending
+ * DistCp's function with copy progress reporting to ZooKeeper implementation.
+ *
+ * For now this is only a wrapper. The other features such as progress and increment backup will be
+ * implemented in future jira
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MapReduceBackupCopyService implements BackupCopyService {
+ private static final Log LOG = LogFactory.getLog(MapReduceBackupCopyService.class);
+
+ private Configuration conf;
+ // private static final long BYTES_PER_MAP = 2 * 256 * 1024 * 1024;
+
+ // Accumulated progress within the whole backup process for the copy operation
+ private float progressDone = 0.1f;
+ private long bytesCopied = 0;
+ private static float INIT_PROGRESS = 0.1f;
+
+ // The percentage of the current copy task within the whole task if multiple time copies are
+ // needed. The default value is 100%, which means only 1 copy task for the whole.
+ private float subTaskPercntgInWholeTask = 1f;
+
+ public MapReduceBackupCopyService() {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Get the current copy task percentage within the whole task if multiple copies are needed.
+ * @return the current copy task percentage
+ */
+ public float getSubTaskPercntgInWholeTask() {
+ return subTaskPercntgInWholeTask;
+ }
+
+ /**
+ * Set the current copy task percentage within the whole task if multiple copies are needed. Must
+ * be called before calling
+ * {@link #copy(BackupHandler, Configuration, Type, String[])}
+ * @param subTaskPercntgInWholeTask The percentage of the copy subtask
+ */
+ public void setSubTaskPercntgInWholeTask(float subTaskPercntgInWholeTask) {
+ this.subTaskPercntgInWholeTask = subTaskPercntgInWholeTask;
+ }
+
+ class SnapshotCopy extends ExportSnapshot {
+ private BackupHandler backupHandler;
+ private String table;
+
+ public SnapshotCopy(BackupHandler backupHandler, String table) {
+ super();
+ this.backupHandler = backupHandler;
+ this.table = table;
+ }
+
+ public BackupHandler getBackupHandler() {
+ return this.backupHandler;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+ }
+
+ // Extends DistCp for progress updating to hbase:backup
+ // during backup. Using DistCpV2 (MAPREDUCE-2765).
+ // Simply extend it and override execute() method to get the
+ // Job reference for progress updating.
+ // Only the argument "src1, [src2, [...]] dst" is supported,
+ // no more DistCp options.
+ class BackupDistCp extends DistCp {
+
+ private BackupHandler backupHandler;
+
+ public BackupDistCp(Configuration conf, DistCpOptions options, BackupHandler backupHandler)
+ throws Exception {
+ super(conf, options);
+ this.backupHandler = backupHandler;
+ }
+
+ @Override
+ public Job execute() throws Exception {
+
+ // reflection preparation for private methods and fields
+ Class<?> classDistCp = org.apache.hadoop.tools.DistCp.class;
+ Method methodCreateMetaFolderPath = classDistCp.getDeclaredMethod("createMetaFolderPath");
+ Method methodCreateJob = classDistCp.getDeclaredMethod("createJob");
+ Method methodCreateInputFileListing =
+ classDistCp.getDeclaredMethod("createInputFileListing", Job.class);
+ Method methodCleanup = classDistCp.getDeclaredMethod("cleanup");
+
+ Field fieldInputOptions = classDistCp.getDeclaredField("inputOptions");
+ Field fieldMetaFolder = classDistCp.getDeclaredField("metaFolder");
+ Field fieldJobFS = classDistCp.getDeclaredField("jobFS");
+ Field fieldSubmitted = classDistCp.getDeclaredField("submitted");
+
+ methodCreateMetaFolderPath.setAccessible(true);
+ methodCreateJob.setAccessible(true);
+ methodCreateInputFileListing.setAccessible(true);
+ methodCleanup.setAccessible(true);
+
+ fieldInputOptions.setAccessible(true);
+ fieldMetaFolder.setAccessible(true);
+ fieldJobFS.setAccessible(true);
+ fieldSubmitted.setAccessible(true);
+
+ // execute() logic starts here
+ assert fieldInputOptions.get(this) != null;
+ assert getConf() != null;
+
+ Job job = null;
+ try {
+ synchronized (this) {
+ // Don't cleanup while we are setting up.
+ fieldMetaFolder.set(this, methodCreateMetaFolderPath.invoke(this));
+ fieldJobFS.set(this, ((Path) fieldMetaFolder.get(this)).getFileSystem(getConf()));
+
+ job = (Job) methodCreateJob.invoke(this);
+ }
+ methodCreateInputFileListing.invoke(this, job);
+
+ // Get the total length of the source files
+ List<Path> srcs = ((DistCpOptions) fieldInputOptions.get(this)).getSourcePaths();
+ long totalSrcLgth = 0;
+ for (Path aSrc : srcs) {
+ totalSrcLgth += BackupUtil.getFilesLength(aSrc.getFileSystem(getConf()), aSrc);
+ }
+
+ // submit the copy job
+ job.submit();
+ fieldSubmitted.set(this, true);
+
+ // after submit the MR job, set its handler in backup handler for cancel process
+ // this.backupHandler.copyJob = job;
+
+ // Update the copy progress to ZK every 0.5s if progress value changed
+ int progressReportFreq =
+ this.getConf().getInt("hbase.backup.progressreport.frequency", 500);
+ float lastProgress = progressDone;
+ while (!job.isComplete()) {
+ float newProgress =
+ progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS);
+
+ if (newProgress > lastProgress) {
+
+ BigDecimal progressData =
+ new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP);
+ String newProgressStr = progressData + "%";
+ LOG.info("Progress: " + newProgressStr);
+ this.backupHandler.updateProgress(newProgressStr, bytesCopied);
+ LOG.debug("Backup progress data updated to hbase:backup: \"Progress: " + newProgressStr
+ + ".\"");
+ lastProgress = newProgress;
+ }
+ Thread.sleep(progressReportFreq);
+ }
+
+ // update the progress data after copy job complete
+ float newProgress =
+ progressDone + job.mapProgress() * subTaskPercntgInWholeTask * (1 - INIT_PROGRESS);
+ BigDecimal progressData =
+ new BigDecimal(newProgress * 100).setScale(1, BigDecimal.ROUND_HALF_UP);
+
+ String newProgressStr = progressData + "%";
+ LOG.info("Progress: " + newProgressStr);
+
+ // accumulate the overall backup progress
+ progressDone = newProgress;
+ bytesCopied += totalSrcLgth;
+
+ this.backupHandler.updateProgress(newProgressStr, bytesCopied);
+ LOG.debug("Backup progress data updated to hbase:backup: \"Progress: " + newProgressStr
+ + " - " + bytesCopied + " bytes copied.\"");
+
+ } finally {
+ if (!fieldSubmitted.getBoolean(this)) {
+ methodCleanup.invoke(this);
+ }
+ }
+
+ String jobID = job.getJobID().toString();
+ job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, jobID);
+
+ LOG.debug("DistCp job-id: " + jobID);
+ return job;
+ }
+
+ }
+
+ /**
+ * Do backup copy based on different types.
+ * @param handler The backup handler reference
+ * @param conf The hadoop configuration
+ * @param copyType The backup copy type
+ * @param options Options for customized ExportSnapshot or DistCp
+ * @throws Exception exception
+ */
+ public int copy(BackupHandler handler, Configuration conf, BackupCopyService.Type copyType,
+ String[] options) throws IOException {
+
+ int res = 0;
+
+ try {
+ if (copyType == Type.FULL) {
+ SnapshotCopy snapshotCp =
+ new SnapshotCopy(handler, handler.getBackupContext().getTableBySnapshot(options[1]));
+ LOG.debug("Doing SNAPSHOT_COPY");
+ // Make a new instance of conf to be used by the snapshot copy class.
+ snapshotCp.setConf(new Configuration(conf));
+ res = snapshotCp.run(options);
+ } else if (copyType == Type.INCREMENTAL) {
+ LOG.debug("Doing COPY_TYPE_DISTCP");
+ setSubTaskPercntgInWholeTask(1f);
+
+ BackupDistCp distcp = new BackupDistCp(new Configuration(conf), null, handler);
+ // Handle a special case where the source file is a single file.
+ // In this case, distcp will not create the target dir. It just take the
+ // target as a file name and copy source file to the target (as a file name).
+ // We need to create the target dir before run distcp.
+ LOG.debug("DistCp options: " + Arrays.toString(options));
+ if (options.length == 2) {
+ Path dest = new Path(options[1]);
+ FileSystem destfs = dest.getFileSystem(conf);
+ if (!destfs.exists(dest)) {
+ destfs.mkdirs(dest);
+ }
+ }
+
+ res = distcp.run(options);
+ }
+ return res;
+
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java
new file mode 100644
index 0000000..deefbf7
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreService.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.backup.HBackupFileSystem;
+import org.apache.hadoop.hbase.backup.IncrementalRestoreService;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.mapreduce.WALPlayer;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class MapReduceRestoreService implements IncrementalRestoreService {
+ public static final Log LOG = LogFactory.getLog(MapReduceRestoreService.class);
+
+ private WALPlayer player;
+
+ public MapReduceRestoreService() {
+ this.player = new WALPlayer();
+ }
+
+ @Override
+ public void run(String logDir, String[] tableNames, String[] newTableNames) throws IOException {
+ String tableStr = HBackupFileSystem.join(tableNames);
+ String newTableStr = HBackupFileSystem.join(newTableNames);
+
+ // WALPlayer reads all files in arbitrary directory structure and creates a Map task for each
+ // log file
+
+ String[] playerArgs = { logDir, tableStr, newTableStr };
+ LOG.info("Restore incremental backup from directory " + logDir + " from hbase tables "
+ + HBackupFileSystem.join(tableNames) + " to tables "
+ + HBackupFileSystem.join(newTableNames));
+ try {
+ player.run(playerArgs);
+ } catch (Exception e) {
+ throw new IOException("cannot restore from backup directory " + logDir
+ + " (check Hadoop and HBase logs) " + e);
+ }
+ }
+
+ @Override
+ public Configuration getConf() {
+ return player.getConf();
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.player.setConf(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
new file mode 100644
index 0000000..4712548
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/BackupLogCleaner.java
@@ -0,0 +1,121 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.master;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.backup.BackupSystemTable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
+
+
+
+
+/**
+ * Implementation of a log cleaner that checks if a log is still scheduled for
+ * incremental backup before deleting it when its TTL is over.
+ */
+@InterfaceStability.Evolving
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class BackupLogCleaner extends BaseLogCleanerDelegate {
+ private static final Log LOG = LogFactory.getLog(BackupLogCleaner.class);
+
+ private boolean stopped = false;
+
+ public BackupLogCleaner() {
+ }
+
+ @Override
+ public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
+ // all members of this class are null if backup is disabled,
+ // so we cannot filter the files
+ if (this.getConf() == null) {
+ return files;
+ }
+
+ try {
+ final BackupSystemTable table = BackupSystemTable.getTable(getConf());
+ // If we do not have recorded backup sessions
+ if (table.hasBackupSessions() == false) {
+ return files;
+ }
+ return Iterables.filter(files, new Predicate<FileStatus>() {
+ @Override
+ public boolean apply(FileStatus file) {
+ try {
+ String wal = file.getPath().toString();
+ boolean logInSystemTable = table.checkWALFile(wal);
+ if (LOG.isDebugEnabled()) {
+ if (logInSystemTable) {
+ LOG.debug("Found log file in hbase:backup, deleting: " + wal);
+ } else {
+ LOG.debug("Didn't find this log in hbase:backup, keeping: " + wal);
+ }
+ }
+ return logInSystemTable;
+ } catch (IOException e) {
+ LOG.error(e);
+ return false;// keep file for a while, HBase failed
+ }
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("Failed to get hbase:backup table, therefore will keep all files", e);
+ // nothing to delete
+ return new ArrayList<FileStatus>();
+ }
+
+ }
+
+ @Override
+ public void setConf(Configuration config) {
+ // If backup is disabled, keep all members null
+ if (!config.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT)) {
+ LOG.warn("Backup is disabled - allowing all wals to be deleted");
+ return;
+ }
+ super.setConf(config);
+ }
+
+ @Override
+ public void stop(String why) {
+ if (this.stopped) {
+ return;
+ }
+ this.stopped = true;
+ LOG.info("Stopping BackupLogCleaner");
+ }
+
+ @Override
+ public boolean isStopped() {
+ return this.stopped;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
new file mode 100644
index 0000000..f96682f
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.master;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MetricsMaster;
+import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
+import org.apache.zookeeper.KeeperException;
+
+public class LogRollMasterProcedureManager extends MasterProcedureManager {
+
+ public static final String ROLLLOG_PROCEDURE_SIGNATURE = "rolllog-proc";
+ public static final String ROLLLOG_PROCEDURE_NAME = "rolllog";
+ private static final Log LOG = LogFactory.getLog(LogRollMasterProcedureManager.class);
+
+ private MasterServices master;
+ private ProcedureCoordinator coordinator;
+ private boolean done;
+
+ @Override
+ public void stop(String why) {
+ LOG.info("stop: " + why);
+ }
+
+ @Override
+ public boolean isStopped() {
+ return false;
+ }
+
+ @Override
+ public void initialize(MasterServices master, MetricsMaster metricsMaster)
+ throws KeeperException, IOException, UnsupportedOperationException {
+ this.master = master;
+ this.done = false;
+
+ // setup the default procedure coordinator
+ String name = master.getServerName().toString();
+ ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, 1);
+ BaseCoordinatedStateManager coordManager =
+ (BaseCoordinatedStateManager) CoordinatedStateManagerFactory
+ .getCoordinatedStateManager(master.getConfiguration());
+ coordManager.initialize(master);
+
+ ProcedureCoordinatorRpcs comms =
+ coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(), name);
+
+ this.coordinator = new ProcedureCoordinator(comms, tpool);
+ }
+
+ @Override
+ public String getProcedureSignature() {
+ return ROLLLOG_PROCEDURE_SIGNATURE;
+ }
+
+ @Override
+ public void execProcedure(ProcedureDescription desc) throws IOException {
+ this.done = false;
+ // start the process on the RS
+ ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getInstance());
+ List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
+ List<String> servers = new ArrayList<String>();
+ for (ServerName sn : serverNames) {
+ servers.add(sn.toString());
+ }
+ Procedure proc = coordinator.startProcedure(monitor, desc.getInstance(), new byte[0], servers);
+ if (proc == null) {
+ String msg = "Failed to submit distributed procedure for '" + desc.getInstance() + "'";
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+
+ try {
+ // wait for the procedure to complete. A timer thread is kicked off that should cancel this
+ // if it takes too long.
+ proc.waitForCompleted();
+ LOG.info("Done waiting - exec procedure for " + desc.getInstance());
+ LOG.info("Distributed roll log procedure is successful!");
+ this.done = true;
+ } catch (InterruptedException e) {
+ ForeignException ee =
+ new ForeignException("Interrupted while waiting for roll log procdure to finish", e);
+ monitor.receive(ee);
+ Thread.currentThread().interrupt();
+ } catch (ForeignException e) {
+ ForeignException ee =
+ new ForeignException("Exception while waiting for roll log procdure to finish", e);
+ monitor.receive(ee);
+ }
+ monitor.rethrowException();
+ }
+
+ @Override
+ public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
+ return done;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
new file mode 100644
index 0000000..618748e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.regionserver;
+
+import java.util.HashMap;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.backup.BackupSystemTable;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.procedure.ProcedureMember;
+import org.apache.hadoop.hbase.procedure.Subprocedure;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
+
+
+/**
+ * This backup subprocedure implementation forces a log roll on the RS.
+ */
+public class LogRollBackupSubprocedure extends Subprocedure {
+ private static final Log LOG = LogFactory.getLog(LogRollBackupSubprocedure.class);
+
+ private final RegionServerServices rss;
+ private final LogRollBackupSubprocedurePool taskManager;
+ private FSHLog hlog;
+
+ public LogRollBackupSubprocedure(RegionServerServices rss, ProcedureMember member,
+ ForeignExceptionDispatcher errorListener, long wakeFrequency, long timeout,
+ LogRollBackupSubprocedurePool taskManager) {
+
+ super(member, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, errorListener,
+ wakeFrequency, timeout);
+ LOG.info("Constructing a LogRollBackupSubprocedure.");
+ this.rss = rss;
+ this.taskManager = taskManager;
+ }
+
+ /**
+ * Callable task. TODO. We don't need a thread pool to execute roll log. This can be simplified
+ * with no use of subprocedurepool.
+ */
+ class RSRollLogTask implements Callable<Void> {
+ RSRollLogTask() {
+ }
+
+ @Override
+ public Void call() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("++ DRPC started: " + rss.getServerName());
+ }
+ hlog = (FSHLog) rss.getWAL(null);
+ long filenum = hlog.getFilenum();
+
+ LOG.info("Trying to roll log in backup subprocedure, current log number: " + filenum);
+ hlog.rollWriter(true);
+ LOG.info("After roll log in backup subprocedure, current log number: " + hlog.getFilenum());
+ // write the log number to hbase:backup.
+ BackupSystemTable table = BackupSystemTable.getTable(rss.getConfiguration());
+ // sanity check, good for testing
+ HashMap<String, String> serverTimestampMap = table.readRegionServerLastLogRollResult();
+ String host = rss.getServerName().getHostname();
+ String sts = serverTimestampMap.get(host);
+ if (sts != null && Long.parseLong(sts) > filenum) {
+ LOG.warn("Won't update server's last roll log result: current=" + sts + " new=" + filenum);
+ return null;
+ }
+ table.writeRegionServerLastLogRollResult(host, Long.toString(filenum));
+ // TODO: potential leak of HBase connection
+ // BackupSystemTable.close();
+ return null;
+ }
+
+ }
+
+ private void rolllog() throws ForeignException {
+
+ monitor.rethrowException();
+
+ taskManager.submitTask(new RSRollLogTask());
+ monitor.rethrowException();
+
+ // wait for everything to complete.
+ taskManager.waitForOutstandingTasks();
+ monitor.rethrowException();
+
+ }
+
+ @Override
+ public void acquireBarrier() throws ForeignException {
+ // do nothing, executing in inside barrier step.
+ }
+
+ /**
+ * do a log roll.
+ * @return some bytes
+ */
+ @Override
+ public byte[] insideBarrier() throws ForeignException {
+ rolllog();
+ // FIXME
+ return null;
+ }
+
+ /**
+ * Cancel threads if they haven't finished.
+ */
+ @Override
+ public void cleanup(Exception e) {
+ taskManager.abort("Aborting log roll subprocedure tasks for backup due to error", e);
+ }
+
+ /**
+ * Hooray!
+ */
+ public void releaseBarrier() {
+ // NO OP
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java
new file mode 100644
index 0000000..1ca638c
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedurePool.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.regionserver;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.DaemonThreadFactory;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+
+/**
+ * Handle running each of the individual tasks for completing a backup procedure
+ * on a regionserver.
+ */
+public class LogRollBackupSubprocedurePool implements Closeable, Abortable {
+ private static final Log LOG = LogFactory.getLog(LogRollBackupSubprocedurePool.class);
+
+ /** Maximum number of concurrent snapshot region tasks that can run concurrently */
+ private static final String CONCURENT_BACKUP_TASKS_KEY = "hbase.backup.region.concurrentTasks";
+ private static final int DEFAULT_CONCURRENT_BACKUP_TASKS = 3;
+
+ private final ExecutorCompletionService<Void> taskPool;
+ private final ThreadPoolExecutor executor;
+ private volatile boolean aborted;
+ private final List<Future<Void>> futures = new ArrayList<Future<Void>>();
+ private final String name;
+
+ public LogRollBackupSubprocedurePool(String name, Configuration conf) {
+ // configure the executor service
+ long keepAlive =
+ conf.getLong(LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_KEY,
+ LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_DEFAULT);
+ int threads = conf.getInt(CONCURENT_BACKUP_TASKS_KEY, DEFAULT_CONCURRENT_BACKUP_TASKS);
+ this.name = name;
+ executor =
+ new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory("rs(" + name
+ + ")-backup-pool"));
+ taskPool = new ExecutorCompletionService<Void>(executor);
+ }
+
+ /**
+ * Submit a task to the pool.
+ */
+ public void submitTask(final Callable<Void> task) {
+ Future<Void> f = this.taskPool.submit(task);
+ futures.add(f);
+ }
+
+ /**
+ * Wait for all of the currently outstanding tasks submitted via {@link #submitTask(Callable)}
+ * @return <tt>true</tt> on success, <tt>false</tt> otherwise
+ * @throws ForeignException exception
+ */
+ public boolean waitForOutstandingTasks() throws ForeignException {
+ LOG.debug("Waiting for backup procedure to finish.");
+
+ try {
+ for (Future<Void> f : futures) {
+ f.get();
+ }
+ return true;
+ } catch (InterruptedException e) {
+ if (aborted) {
+ throw new ForeignException("Interrupted and found to be aborted while waiting for tasks!",
+ e);
+ }
+ Thread.currentThread().interrupt();
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof ForeignException) {
+ throw (ForeignException) e.getCause();
+ }
+ throw new ForeignException(name, e.getCause());
+ } finally {
+ // close off remaining tasks
+ for (Future<Void> f : futures) {
+ if (!f.isDone()) {
+ f.cancel(true);
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Attempt to cleanly shutdown any running tasks - allows currently running tasks to cleanly
+ * finish
+ */
+ @Override
+ public void close() {
+ executor.shutdown();
+ }
+
+ @Override
+ public void abort(String why, Throwable e) {
+ if (this.aborted) {
+ return;
+ }
+
+ this.aborted = true;
+ LOG.warn("Aborting because: " + why, e);
+ this.executor.shutdownNow();
+ }
+
+ @Override
+ public boolean isAborted() {
+ return this.aborted;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
new file mode 100644
index 0000000..aca190c
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollRegionServerProcedureManager.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.regionserver;
+
+
+import java.io.IOException;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.procedure.ProcedureMember;
+import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
+import org.apache.hadoop.hbase.procedure.RegionServerProcedureManager;
+import org.apache.hadoop.hbase.procedure.Subprocedure;
+import org.apache.hadoop.hbase.procedure.SubprocedureFactory;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * This manager class handles the work dealing with backup for a {@link HRegionServer}.
+ * <p>
+ * This provides the mechanism necessary to kick off a backup specific {@link Subprocedure} that is
+ * responsible by this region server. If any failures occur with the subprocedure, the manager's
+ * procedure member notifies the procedure coordinator to abort all others.
+ * <p>
+ * On startup, requires {@link #start()} to be called.
+ * <p>
+ * On shutdown, requires org.apache.hadoop.hbase.procedure.ProcedureMember.close() to be
+ * called
+ */
+public class LogRollRegionServerProcedureManager extends RegionServerProcedureManager {
+
+ private static final Log LOG = LogFactory.getLog(LogRollRegionServerProcedureManager.class);
+
+ /** Conf key for number of request threads to start backup on regionservers */
+ public static final String BACKUP_REQUEST_THREADS_KEY = "hbase.backup.region.pool.threads";
+ /** # of threads for backup work on the rs. */
+ public static final int BACKUP_REQUEST_THREADS_DEFAULT = 10;
+
+ public static final String BACKUP_TIMEOUT_MILLIS_KEY = "hbase.backup.timeout";
+ public static final long BACKUP_TIMEOUT_MILLIS_DEFAULT = 60000;
+
+ /** Conf key for millis between checks to see if backup work completed or if there are errors */
+ public static final String BACKUP_REQUEST_WAKE_MILLIS_KEY = "hbase.backup.region.wakefrequency";
+ /** Default amount of time to check for errors while regions finish backup work */
+ private static final long BACKUP_REQUEST_WAKE_MILLIS_DEFAULT = 500;
+
+ private RegionServerServices rss;
+ private ProcedureMemberRpcs memberRpcs;
+ private ProcedureMember member;
+
+ /**
+ * Create a default backup procedure manager
+ */
+ public LogRollRegionServerProcedureManager() {
+ }
+
+ /**
+ * Start accepting backup procedure requests.
+ */
+ @Override
+ public void start() {
+ this.memberRpcs.start(rss.getServerName().toString(), member);
+ LOG.info("Started region server backup manager.");
+ }
+
+ /**
+ * Close <tt>this</tt> and all running backup procedure tasks
+ * @param force forcefully stop all running tasks
+ * @throws IOException exception
+ */
+ @Override
+ public void stop(boolean force) throws IOException {
+ String mode = force ? "abruptly" : "gracefully";
+ LOG.info("Stopping RegionServerBackupManager " + mode + ".");
+
+ try {
+ this.member.close();
+ } finally {
+ this.memberRpcs.close();
+ }
+ }
+
+ /**
+ * If in a running state, creates the specified subprocedure for handling a backup procedure.
+ * @return Subprocedure to submit to the ProcedureMemeber.
+ */
+ public Subprocedure buildSubprocedure() {
+
+ // don't run a backup if the parent is stop(ping)
+ if (rss.isStopping() || rss.isStopped()) {
+ throw new IllegalStateException("Can't start backup procedure on RS: " + rss.getServerName()
+ + ", because stopping/stopped!");
+ }
+
+ LOG.info("Attempting to run a roll log procedure for backup.");
+ ForeignExceptionDispatcher errorDispatcher = new ForeignExceptionDispatcher();
+ Configuration conf = rss.getConfiguration();
+ long timeoutMillis = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT);
+ long wakeMillis =
+ conf.getLong(BACKUP_REQUEST_WAKE_MILLIS_KEY, BACKUP_REQUEST_WAKE_MILLIS_DEFAULT);
+
+ LogRollBackupSubprocedurePool taskManager =
+ new LogRollBackupSubprocedurePool(rss.getServerName().toString(), conf);
+ return new LogRollBackupSubprocedure(rss, member, errorDispatcher, wakeMillis, timeoutMillis,
+ taskManager);
+
+ }
+
+ /**
+ * Build the actual backup procedure runner that will do all the 'hard' work
+ */
+ public class BackupSubprocedureBuilder implements SubprocedureFactory {
+
+ @Override
+ public Subprocedure buildSubprocedure(String name, byte[] data) {
+ return LogRollRegionServerProcedureManager.this.buildSubprocedure();
+ }
+ }
+
+ @Override
+ public void initialize(RegionServerServices rss) throws IOException {
+ this.rss = rss;
+ BaseCoordinatedStateManager coordManager =
+ (BaseCoordinatedStateManager) CoordinatedStateManagerFactory.getCoordinatedStateManager(rss
+ .getConfiguration());
+ coordManager.initialize(rss);
+ this.memberRpcs =
+ coordManager
+ .getProcedureMemberRpcs(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE);
+
+ // read in the backup handler configuration properties
+ Configuration conf = rss.getConfiguration();
+ long keepAlive = conf.getLong(BACKUP_TIMEOUT_MILLIS_KEY, BACKUP_TIMEOUT_MILLIS_DEFAULT);
+ int opThreads = conf.getInt(BACKUP_REQUEST_THREADS_KEY, BACKUP_REQUEST_THREADS_DEFAULT);
+ // create the actual cohort member
+ ThreadPoolExecutor pool =
+ ProcedureMember.defaultPool(rss.getServerName().toString(), opThreads, keepAlive);
+ this.member = new ProcedureMember(memberRpcs, pool, new BackupSubprocedureBuilder());
+ }
+
+ @Override
+ public String getProcedureSignature() {
+ return "backup-proc";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
index ae36f08..3342743 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/BaseCoordinatedStateManager.java
@@ -17,7 +17,11 @@
*/
package org.apache.hadoop.hbase.coordination;
+import java.io.IOException;
+
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.Server;
@@ -51,8 +55,21 @@ public abstract class BaseCoordinatedStateManager implements CoordinatedStateMan
* Method to retrieve coordination for split log worker
*/
public abstract SplitLogWorkerCoordination getSplitLogWorkerCoordination();
+
/**
* Method to retrieve coordination for split log manager
*/
public abstract SplitLogManagerCoordination getSplitLogManagerCoordination();
+ /**
+ * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs}
+ */
+ public abstract ProcedureCoordinatorRpcs
+ getProcedureCoordinatorRpcs(String procType, String coordNode) throws IOException;
+
+ /**
+ * Method to retrieve {@link org.apache.hadoop.hbase.procedure.ProcedureMemberRpc}
+ */
+ public abstract ProcedureMemberRpcs
+ getProcedureMemberRpcs(String procType) throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
index 3e89be7..7cf4aab 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkCoordinatedStateManager.java
@@ -17,9 +17,15 @@
*/
package org.apache.hadoop.hbase.coordination;
+import java.io.IOException;
+
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.ProcedureMemberRpcs;
+import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.ZKProcedureMemberRpcs;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/**
@@ -49,9 +55,21 @@ public class ZkCoordinatedStateManager extends BaseCoordinatedStateManager {
@Override
public SplitLogWorkerCoordination getSplitLogWorkerCoordination() {
return splitLogWorkerCoordination;
- }
+ }
+
@Override
public SplitLogManagerCoordination getSplitLogManagerCoordination() {
return splitLogManagerCoordination;
}
+
+ @Override
+ public ProcedureCoordinatorRpcs getProcedureCoordinatorRpcs(String procType, String coordNode)
+ throws IOException {
+ return new ZKProcedureCoordinatorRpcs(watcher, procType, coordNode);
+ }
+
+ @Override
+ public ProcedureMemberRpcs getProcedureMemberRpcs(String procType) throws IOException {
+ return new ZKProcedureMemberRpcs(watcher, procType);
+ }
}
[26/29] hbase git commit: HBASE-14888 ClusterSchema: Add Namespace
Operations
Posted by sy...@apache.org.
HBASE-14888 ClusterSchema: Add Namespace Operations
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c5f3d17a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c5f3d17a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c5f3d17a
Branch: refs/heads/hbase-12439
Commit: c5f3d17ae3a61cbf77cab89cddd8303e20e5e734
Parents: f01a388
Author: stack <st...@apache.org>
Authored: Sat Jan 2 16:16:15 2016 -0800
Committer: stack <st...@apache.org>
Committed: Sat Jan 2 16:16:56 2016 -0800
----------------------------------------------------------------------
.../hbase/client/ConnectionImplementation.java | 2 +-
.../java/org/apache/hadoop/hbase/Service.java | 50 +++
.../hbase/ServiceNotRunningException.java | 39 +++
.../apache/hadoop/hbase/ResourceChecker.java | 2 +-
.../hadoop/hbase/master/ClusterSchema.java | 153 +++++++++
.../hbase/master/ClusterSchemaException.java | 37 +++
.../hbase/master/ClusterSchemaService.java | 27 ++
.../hbase/master/ClusterSchemaServiceImpl.java | 164 ++++++++++
.../org/apache/hadoop/hbase/master/HMaster.java | 314 ++++++-------------
.../hadoop/hbase/master/MasterRpcServices.java | 10 +-
.../hadoop/hbase/master/MasterServices.java | 78 +----
.../hadoop/hbase/master/ProcedureFuture.java | 132 ++++++++
.../hbase/master/TableNamespaceManager.java | 68 ++--
.../procedure/CreateNamespaceProcedure.java | 2 +-
.../procedure/DeleteNamespaceProcedure.java | 2 +-
.../procedure/ModifyNamespaceProcedure.java | 3 +-
.../hbase/namespace/NamespaceStateManager.java | 6 +-
.../hbase/regionserver/HRegionServer.java | 3 +-
.../org/apache/hadoop/hbase/TestNamespace.java | 2 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 61 +---
.../hbase/master/TestMasterNoCluster.java | 8 -
21 files changed, 762 insertions(+), 401 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 0ef2a17..3498590 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -398,7 +398,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
synchronized (this) {
if (batchPool == null) {
this.batchPool = getThreadPool(conf.getInt("hbase.hconnection.threads.max", 256),
- conf.getInt("hbase.hconnection.threads.core", 256), "-shared-", null);
+ conf.getInt("hbase.hconnection.threads.core", 256), "-shared", null);
this.cleanupPool = true;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java
new file mode 100644
index 0000000..97d93cc
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Service.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Simple Service.
+ */
+// This is a WIP. We have Services throughout hbase. Either have all implement what is here or
+// just remove this as an experiment that did not work out.
+// TODO: Move on to guava Service after we update our guava version; later guava has nicer
+// Service implmentation.
+// TODO: Move all Services on to this one Interface.
+@InterfaceAudience.Private
+public interface Service {
+ /**
+ * Initiates service startup (if necessary), returning once the service has finished starting.
+ * @throws IOException Throws exception if already running and if we fail to start successfully.
+ */
+ void startAndWait() throws IOException;
+
+ /**
+ * @return True if this Service is running.
+ */
+ boolean isRunning();
+
+ /**
+ * Initiates service shutdown (if necessary), returning once the service has finished stopping.
+ * @throws IOException Throws exception if not running of if we fail to stop successfully.
+ */
+ void stopAndWait() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java
new file mode 100644
index 0000000..f6325ea
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServiceNotRunningException.java
@@ -0,0 +1,39 @@
+package org.apache.hadoop.hbase;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+public class ServiceNotRunningException extends HBaseIOException {
+ public ServiceNotRunningException() {
+ }
+
+ public ServiceNotRunningException(String message) {
+ super(message);
+ }
+
+ public ServiceNotRunningException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ServiceNotRunningException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
index 539aea3..ee0380a 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
@@ -41,7 +41,7 @@ public class ResourceChecker {
/**
* Constructor
- * @param tagLine - the tagLine is added to the logs. Must be be null.
+ * @param tagLine The tagLine is added to the logs. Must not be null.
*/
public ResourceChecker(final String tagLine) {
this.tagLine = tagLine;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java
new file mode 100644
index 0000000..e632230
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * View and edit the current cluster schema. Use this API making any modification to
+ * namespaces, tables, etc.
+ *
+ * <h2>Implementation Notes</h2>
+ * Nonces are for when operation is non-idempotent to ensure once-only semantic, even
+ * across process failures.
+ */
+// ClusterSchema is introduced to encapsulate schema modification. Currently the different aspects
+// are spread about the code base. This effort is about cleanup, shutting down access, and
+// coalescing common code. In particular, we'd contain filesystem modification. Other
+// benefits are to make all schema modification work the same way (one way to do an operation only
+// rather than the current approach where how an operation is done varies with context) and to make
+// it so clusterschema modification can stand apart from Master to faciliate standalone
+// testing. It is part of the filesystem refactor project that undoes the dependency on a
+// layout in HDFS that mimics our model of tables have regions have column families have files.
+// With this Interface in place, with all modifications going via this route where no filesystem
+// particulars are exposed, redoing our internals will take less effort.
+//
+// Currently ClusterSchema Interface will include namespace and table manipulation. Ideally a
+// form of this Interface will go all the ways down to the file manipulation level but currently
+// TBD.
+//
+// ClusterSchema is private to the Master; only the Master knows current cluster state and has
+// means of editing/altering it.
+//
+// TODO: Remove Server argument when MasterServices are passed.
+// TODO: We return Future<ProcedureInfo> in the below from most methods. It may change to return
+// a ProcedureFuture subsequently.
+@InterfaceAudience.Private
+public interface ClusterSchema {
+ /**
+ * Timeout for cluster operations in milliseconds.
+ */
+ public static final String HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY =
+ "hbase.master.cluster.schema.operation.timeout";
+ /**
+ * Default operation timeout in milliseconds.
+ */
+ public static final int DEFAULT_HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT =
+ 5 * 60 * 1000;
+
+ /**
+ * Utility method that will wait {@link #HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY}
+ * timeout and if exceptions, THROWs the exception doing conversion so palatable outside Master:
+ * i.e. {@link InterruptedException} becomes {@link InterruptedIOException} and so on.
+ *
+ * <<Utility>>
+ *
+ * @param future Future to wait on.
+ * @return On completion, info on the procedure that ran.
+ * @throws IOException
+ */
+ // TODO: Where to put this utility? It goes away?
+ ProcedureInfo get(final Future<ProcedureInfo> future) throws IOException;
+
+ /**
+ * For internals use only. Do not use! Provisionally part of this Interface.
+ * Prefer the high-level APIs available elsewhere in this API.
+ * @return Instance of {@link TableNamespaceManager}
+ */
+ // TODO: Remove from here. Keep internal. This Interface is too high-level to host this accessor.
+ TableNamespaceManager getTableNamespaceManager();
+
+ /**
+ * Create a new Namespace.
+ * @param namespaceDescriptor descriptor for new Namespace
+ * @param nonceGroup Identifier for the source of the request, a client or process.
+ * @param nonce A unique identifier for this operation from the client or process identified by
+ * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
+ * @return Operation Future.
+ * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion.
+ * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
+ * as well as {@link IOException}
+ */
+ Future<ProcedureInfo> createNamespace(NamespaceDescriptor namespaceDescriptor, long nonceGroup,
+ long nonce)
+ throws IOException;
+
+ /**
+ * Modify an existing Namespace.
+ * @param nonceGroup Identifier for the source of the request, a client or process.
+ * @param nonce A unique identifier for this operation from the client or process identified by
+ * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
+ * @return Operation Future.
+ * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion.
+ * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
+ * as well as {@link IOException}
+ */
+ Future<ProcedureInfo> modifyNamespace(NamespaceDescriptor descriptor, long nonceGroup,
+ long nonce)
+ throws IOException;
+
+ /**
+ * Delete an existing Namespace.
+ * Only empty Namespaces (no tables) can be removed.
+ * @param nonceGroup Identifier for the source of the request, a client or process.
+ * @param nonce A unique identifier for this operation from the client or process identified by
+ * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
+ * @return Operation Future.
+ * Use {@link Future#get(long, java.util.concurrent.TimeUnit)} to wait on completion.
+ * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
+ * as well as {@link IOException}
+ */
+ Future<ProcedureInfo> deleteNamespace(String name, long nonceGroup, long nonce)
+ throws IOException;
+
+ /**
+ * Get a Namespace
+ * @param name Name of the Namespace
+ * @return Namespace descriptor for <code>name</code>
+ * @throws IOException Throws {@link ClusterSchemaException} and {@link InterruptedIOException}
+ * as well as {@link IOException}
+ */
+ // No Future here because presumption is that the request will go against cached metadata so
+ // return immediately -- no need of running a Procedure.
+ NamespaceDescriptor getNamespace(String name) throws IOException;
+
+ /**
+ * Get all Namespaces
+ * @return All Namespace descriptors
+ * @throws IOException
+ */
+ List<NamespaceDescriptor> getNamespaces() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java
new file mode 100644
index 0000000..62892b6
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaException.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+public class ClusterSchemaException extends HBaseIOException {
+ public ClusterSchemaException(String message) {
+ super(message);
+ }
+
+ public ClusterSchemaException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ClusterSchemaException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java
new file mode 100644
index 0000000..43353ba
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.Service;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Mixes in ClusterSchema and Service
+ */
+@InterfaceAudience.Private
+public interface ClusterSchemaService extends ClusterSchema, Service {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java
new file mode 100644
index 0000000..c7f4692
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.ServiceNotRunningException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
+import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure;
+import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+
+@InterfaceAudience.Private
+class ClusterSchemaServiceImpl implements ClusterSchemaService {
+ private boolean running = false;
+ private final TableNamespaceManager tableNamespaceManager;
+ private final MasterServices masterServices;
+ private final int clusterSchemaOperationTimeoutInMillis;
+ private final static List<NamespaceDescriptor> EMPTY_NAMESPACE_LIST =
+ Collections.unmodifiableList(new ArrayList<NamespaceDescriptor>(0));
+
+ ClusterSchemaServiceImpl(final MasterServices masterServices) {
+ this.masterServices = masterServices;
+ this.tableNamespaceManager = new TableNamespaceManager(masterServices);
+ this.clusterSchemaOperationTimeoutInMillis = this.masterServices.getConfiguration().
+ getInt(HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT_KEY,
+ DEFAULT_HBASE_MASTER_CLUSTER_SCHEMA_OPERATION_TIMEOUT);
+ }
+
+ // All below are synchronized so consistent view on whether running or not.
+
+ @Override
+ public synchronized boolean isRunning() {
+ return this.running;
+ }
+
+ private synchronized void checkIsRunning() throws ServiceNotRunningException {
+ if (!isRunning()) throw new ServiceNotRunningException();
+ }
+
+ @Override
+ public synchronized void startAndWait() throws IOException {
+ if (isRunning()) throw new IllegalStateException("Already running; cannot double-start.");
+ // Set to running FIRST because tableNamespaceManager start uses this class to do namespace ops
+ this.running = true;
+ this.tableNamespaceManager.start();
+ }
+
+ @Override
+ public synchronized void stopAndWait() throws IOException {
+ checkIsRunning();
+ // You can't stop tableNamespaceManager.
+ this.running = false;
+ }
+
+ @Override
+ public ProcedureInfo get(final Future<ProcedureInfo> future) throws IOException {
+ try {
+ ProcedureInfo pi =
+ future.get(this.clusterSchemaOperationTimeoutInMillis, TimeUnit.MILLISECONDS);
+ // If the procedure got an exception, throw it.
+ if (pi.getException() != null) throw pi.getException();
+ return pi;
+ } catch (ExecutionException ee) {
+ // No cleanup to do... just let the exception out.
+ if (ee.getCause() instanceof IOException) throw (IOException)ee.getCause();
+ else throw new ClusterSchemaException(ee.getCause());
+ } catch (InterruptedException e) {
+ IOException ioe = new InterruptedIOException();
+ ioe.initCause(e);
+ throw ioe;
+ } catch (TimeoutException e) {
+ throw new TimeoutIOException(future.toString());
+ }
+ }
+
+ @Override
+ public TableNamespaceManager getTableNamespaceManager() {
+ return this.tableNamespaceManager;
+ }
+
+ private Future<ProcedureInfo> submitProcedure(final Procedure<?> procedure, long nonceGroup,
+ long nonce)
+ throws ServiceNotRunningException {
+ checkIsRunning();
+ ProcedureExecutor<MasterProcedureEnv> pe = this.masterServices.getMasterProcedureExecutor();
+ long procId = pe.submitProcedure(procedure, nonceGroup, nonce);
+ return new ProcedureFuture(pe, procId);
+ }
+
+ @Override
+ public Future<ProcedureInfo> createNamespace(NamespaceDescriptor namespaceDescriptor,
+ long nonceGroup, long nonce)
+ throws IOException {
+ return submitProcedure(new CreateNamespaceProcedure(
+ this.masterServices.getMasterProcedureExecutor().getEnvironment(), namespaceDescriptor),
+ nonceGroup, nonce);
+ }
+
+ @Override
+ public Future<ProcedureInfo> modifyNamespace(NamespaceDescriptor namespaceDescriptor,
+ long nonceGroup, long nonce)
+ throws IOException {
+ return submitProcedure(new ModifyNamespaceProcedure(
+ this.masterServices.getMasterProcedureExecutor().getEnvironment(), namespaceDescriptor),
+ nonceGroup, nonce);
+ }
+
+ @Override
+ public Future<ProcedureInfo> deleteNamespace(String name, long nonceGroup, long nonce)
+ throws IOException {
+ return submitProcedure(new DeleteNamespaceProcedure(
+ this.masterServices.getMasterProcedureExecutor().getEnvironment(), name),
+ nonceGroup, nonce);
+ }
+
+ @Override
+ public NamespaceDescriptor getNamespace(String name) throws IOException {
+ NamespaceDescriptor nsd = getTableNamespaceManager().get(name);
+ if (nsd == null) throw new NamespaceNotFoundException(name);
+ return nsd;
+ }
+
+ @Override
+ public List<NamespaceDescriptor> getNamespaces() throws IOException {
+ checkIsRunning();
+ Set<NamespaceDescriptor> set = getTableNamespaceManager().list();
+ if (set == null || set.isEmpty()) return EMPTY_NAMESPACE_LIST;
+ List<NamespaceDescriptor> list = new ArrayList<NamespaceDescriptor>(set.size());
+ list.addAll(set);
+ return Collections.unmodifiableList(list);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 8e51f25..f29cad6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -63,7 +64,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.Server;
@@ -97,17 +97,14 @@ import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
import org.apache.hadoop.hbase.master.procedure.AddColumnFamilyProcedure;
-import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
import org.apache.hadoop.hbase.master.procedure.DeleteColumnFamilyProcedure;
-import org.apache.hadoop.hbase.master.procedure.DeleteNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
-import org.apache.hadoop.hbase.master.procedure.ModifyNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
@@ -185,7 +182,7 @@ import com.google.protobuf.Service;
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@SuppressWarnings("deprecation")
-public class HMaster extends HRegionServer implements MasterServices, Server {
+public class HMaster extends HRegionServer implements MasterServices {
private static final Log LOG = LogFactory.getLog(HMaster.class.getName());
/**
@@ -256,8 +253,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
// Tracker for region normalizer state
private RegionNormalizerTracker regionNormalizerTracker;
- /** Namespace stuff */
- private TableNamespaceManager tableNamespaceManager;
+ private ClusterSchemaService clusterSchemaService;
// Metrics for the HMaster
final MetricsMaster metricsMaster;
@@ -368,9 +364,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
* Remaining steps of initialization occur in
* #finishActiveMasterInitialization(MonitoredTask) after
* the master becomes the active one.
- *
- * @throws KeeperException
- * @throws IOException
*/
public HMaster(final Configuration conf, CoordinatedStateManager csm)
throws IOException, KeeperException {
@@ -570,10 +563,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/**
* Initialize all ZK based system trackers.
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
- * @throws CoordinatedStateException
*/
void initializeZKBasedSystemTrackers() throws IOException,
InterruptedException, KeeperException, CoordinatedStateException {
@@ -630,11 +619,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
* <li>Ensure assignment of meta/namespace regions<li>
* <li>Handle either fresh cluster start or master failover</li>
* </ol>
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
- * @throws CoordinatedStateException
*/
private void finishActiveMasterInitialization(MonitoredTask status)
throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
@@ -781,8 +765,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
this.catalogJanitorChore = new CatalogJanitor(this, this);
getChoreService().scheduleChore(catalogJanitorChore);
- status.setStatus("Starting namespace manager");
- initNamespace();
+ status.setStatus("Starting cluster schema service");
+ initClusterSchemaService();
if (this.cpHost != null) {
try {
@@ -848,11 +832,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/**
* Create a {@link ServerManager} instance.
- * @param master
- * @param services
- * @return An instance of {@link ServerManager}
- * @throws org.apache.hadoop.hbase.ZooKeeperConnectionException
- * @throws IOException
*/
ServerManager createServerManager(final Server master,
final MasterServices services)
@@ -888,12 +867,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/**
* Check <code>hbase:meta</code> is assigned. If not, assign it.
- * @param status MonitoredTask
- * @param previouslyFailedMetaRSs
- * @param replicaId
- * @throws InterruptedException
- * @throws IOException
- * @throws KeeperException
*/
void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMetaRSs, int replicaId)
throws InterruptedException, IOException, KeeperException {
@@ -965,10 +938,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
status.setStatus("META assigned.");
}
- void initNamespace() throws IOException {
- //create namespace manager
- tableNamespaceManager = new TableNamespaceManager(this);
- tableNamespaceManager.start();
+ void initClusterSchemaService() throws IOException, InterruptedException {
+ this.clusterSchemaService = new ClusterSchemaServiceImpl(this);
+ this.clusterSchemaService.startAndWait();
+ if (!this.clusterSchemaService.isRunning()) throw new HBaseIOException("Failed start");
}
void initQuotaManager() throws IOException {
@@ -1014,7 +987,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/**
* This function returns a set of region server names under hbase:meta recovering region ZK node
* @return Set of meta server names which were recorded in ZK
- * @throws KeeperException
*/
private Set<ServerName> getPreviouselyFailedMetaServersFromZK() throws KeeperException {
Set<ServerName> result = new HashSet<ServerName>();
@@ -1050,11 +1022,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
return tableStateManager;
}
- @Override
- public TableNamespaceManager getTableNamespaceManager() {
- return tableNamespaceManager;
- }
-
/*
* Start up all services. If any of these threads gets an unhandled exception
* then they just die with a logged message. This should be fine because
@@ -1201,7 +1168,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/**
* @return Get remote side's InetAddress
- * @throws UnknownHostException
*/
InetAddress getRemoteInetAddress(final int port,
final long serverStartCode) throws UnknownHostException {
@@ -1338,7 +1304,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
* @return true if normalization step was performed successfully, false otherwise
* (specifically, if HMaster hasn't been initialized properly or normalization
* is globally disabled)
- * @throws IOException
*/
public boolean normalizeRegions() throws IOException {
if (!this.initialized) {
@@ -1478,9 +1443,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
if (isStopped()) {
throw new MasterNotRunningException();
}
-
+ checkInitialized();
String namespace = hTableDescriptor.getTableName().getNamespaceAsString();
- ensureNamespaceExists(namespace);
+ this.clusterSchemaService.getNamespace(namespace);
HRegionInfo[] newRegions = ModifyRegionUtils.createHRegionInfos(hTableDescriptor, splitKeys);
checkInitialized();
@@ -2305,18 +2270,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
void checkInitialized() throws PleaseHoldException, ServerNotRunningYetException {
checkServiceStarted();
- if (!this.initialized) {
- throw new PleaseHoldException("Master is initializing");
- }
+ if (!isInitialized()) throw new PleaseHoldException("Master is initializing");
}
- void checkNamespaceManagerReady() throws IOException {
- checkInitialized();
- if (tableNamespaceManager == null ||
- !tableNamespaceManager.isTableAvailableAndInitialized(true)) {
- throw new IOException("Table Namespace Manager not ready yet, try again later");
- }
- }
/**
* Report whether this master is currently the active master or not.
* If not active master, we are parked on ZK waiting to become active.
@@ -2411,7 +2367,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
/**
* Utility for constructing an instance of the passed HMaster class.
* @param masterClass
- * @param conf
* @return HMaster instance.
*/
public static HMaster constructMaster(Class<? extends HMaster> masterClass,
@@ -2452,138 +2407,110 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
}
@Override
- public void createNamespace(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException {
- TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
- checkNamespaceManagerReady();
- if (cpHost != null) {
- if (cpHost.preCreateNamespace(descriptor)) {
- return;
- }
- }
- createNamespaceSync(descriptor, nonceGroup, nonce);
- if (cpHost != null) {
- cpHost.postCreateNamespace(descriptor);
- }
+ public ClusterSchema getClusterSchema() {
+ return this.clusterSchemaService;
}
- @Override
- public void createNamespaceSync(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException {
- LOG.info(getClientIdAuditPrefix() + " creating " + descriptor);
+ /**
+ * Create a new Namespace.
+ * @param namespaceDescriptor descriptor for new Namespace
+ * @param nonceGroup Identifier for the source of the request, a client or process.
+ * @param nonce A unique identifier for this operation from the client or process identified by
+ * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
+ */
+ void createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
+ final long nonce)
+ throws IOException {
+ checkInitialized();
+ TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
+ if (this.cpHost != null && this.cpHost.preCreateNamespace(namespaceDescriptor)) return;
+ LOG.info(getClientIdAuditPrefix() + " creating " + namespaceDescriptor);
// Execute the operation synchronously - wait for the operation to complete before continuing.
- long procId = this.procedureExecutor.submitProcedure(
- new CreateNamespaceProcedure(procedureExecutor.getEnvironment(), descriptor),
- nonceGroup,
- nonce);
- ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
+ Future<ProcedureInfo> future =
+ getClusterSchema().createNamespace(namespaceDescriptor, nonceGroup, nonce);
+ getClusterSchema().get(future);
+ if (this.cpHost != null) this.cpHost.postCreateNamespace(namespaceDescriptor);
}
- @Override
- public void modifyNamespace(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException {
- TableName.isLegalNamespaceName(Bytes.toBytes(descriptor.getName()));
- checkNamespaceManagerReady();
- if (cpHost != null) {
- if (cpHost.preModifyNamespace(descriptor)) {
- return;
- }
- }
- LOG.info(getClientIdAuditPrefix() + " modify " + descriptor);
+ /**
+ * Modify an existing Namespace.
+ * @param nonceGroup Identifier for the source of the request, a client or process.
+ * @param nonce A unique identifier for this operation from the client or process identified by
+ * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).\
+ */
+ void modifyNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup,
+ final long nonce)
+ throws IOException {
+ checkInitialized();
+ TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName()));
+ if (this.cpHost != null && this.cpHost.preModifyNamespace(namespaceDescriptor)) return;
+ LOG.info(getClientIdAuditPrefix() + " modify " + namespaceDescriptor);
// Execute the operation synchronously - wait for the operation to complete before continuing.
- long procId = this.procedureExecutor.submitProcedure(
- new ModifyNamespaceProcedure(procedureExecutor.getEnvironment(), descriptor),
- nonceGroup,
- nonce);
- ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
- if (cpHost != null) {
- cpHost.postModifyNamespace(descriptor);
- }
+ Future<ProcedureInfo> future =
+ getClusterSchema().modifyNamespace(namespaceDescriptor, nonceGroup, nonce);
+ getClusterSchema().get(future);
+ if (this.cpHost != null) this.cpHost.postModifyNamespace(namespaceDescriptor);
}
- @Override
- public void deleteNamespace(
- final String name,
- final long nonceGroup,
- final long nonce) throws IOException {
- checkNamespaceManagerReady();
- if (cpHost != null) {
- if (cpHost.preDeleteNamespace(name)) {
- return;
- }
- }
+ /**
+ * Delete an existing Namespace. Only empty Namespaces (no tables) can be removed.
+ * @param nonceGroup Identifier for the source of the request, a client or process.
+ * @param nonce A unique identifier for this operation from the client or process identified by
+ * <code>nonceGroup</code> (the source must ensure each operation gets a unique id).
+ * @throws IOException
+ */
+ void deleteNamespace(final String name, final long nonceGroup, final long nonce)
+ throws IOException {
+ checkInitialized();
+ if (this.cpHost != null && this.cpHost.preDeleteNamespace(name)) return;
LOG.info(getClientIdAuditPrefix() + " delete " + name);
// Execute the operation synchronously - wait for the operation to complete before continuing.
- long procId = this.procedureExecutor.submitProcedure(
- new DeleteNamespaceProcedure(procedureExecutor.getEnvironment(), name),
- nonceGroup,
- nonce);
- ProcedureSyncWait.waitForProcedureToComplete(procedureExecutor, procId);
- if (cpHost != null) {
- cpHost.postDeleteNamespace(name);
- }
+ Future<ProcedureInfo> future = getClusterSchema().deleteNamespace(name, nonceGroup, nonce);
+ getClusterSchema().get(future);
+ if (this.cpHost != null) this.cpHost.postDeleteNamespace(name);
}
/**
- * Ensure that the specified namespace exists, otherwise throws a NamespaceNotFoundException
- *
- * @param name the namespace to check
- * @throws IOException if the namespace manager is not ready yet.
- * @throws NamespaceNotFoundException if the namespace does not exists
+ * Get a Namespace
+ * @param name Name of the Namespace
+ * @return Namespace descriptor for <code>name</code>
*/
- private void ensureNamespaceExists(final String name)
- throws IOException, NamespaceNotFoundException {
- checkNamespaceManagerReady();
- NamespaceDescriptor nsd = tableNamespaceManager.get(name);
- if (nsd == null) {
- throw new NamespaceNotFoundException(name);
- }
+ NamespaceDescriptor getNamespace(String name) throws IOException {
+ checkInitialized();
+ if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name);
+ NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name);
+ if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd);
+ return nsd;
}
- @Override
- public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
- checkNamespaceManagerReady();
-
+ /**
+ * Get all Namespaces
+ * @return All Namespace descriptors
+ */
+ List<NamespaceDescriptor> getNamespaces() throws IOException {
+ checkInitialized();
+ final List<NamespaceDescriptor> nsds = new ArrayList<NamespaceDescriptor>();
+ boolean bypass = false;
if (cpHost != null) {
- cpHost.preGetNamespaceDescriptor(name);
+ bypass = cpHost.preListNamespaceDescriptors(nsds);
}
-
- NamespaceDescriptor nsd = tableNamespaceManager.get(name);
- if (nsd == null) {
- throw new NamespaceNotFoundException(name);
- }
-
- if (cpHost != null) {
- cpHost.postGetNamespaceDescriptor(nsd);
+ if (!bypass) {
+ nsds.addAll(this.clusterSchemaService.getNamespaces());
+ if (this.cpHost != null) this.cpHost.postListNamespaceDescriptors(nsds);
}
-
- return nsd;
+ return nsds;
}
@Override
- public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
- checkNamespaceManagerReady();
-
- final List<NamespaceDescriptor> descriptors = new ArrayList<NamespaceDescriptor>();
- boolean bypass = false;
- if (cpHost != null) {
- bypass = cpHost.preListNamespaceDescriptors(descriptors);
- }
-
- if (!bypass) {
- descriptors.addAll(tableNamespaceManager.list());
+ public List<TableName> listTableNamesByNamespace(String name) throws IOException {
+ checkInitialized();
+ return listTableNames(name, null, true);
+ }
- if (cpHost != null) {
- cpHost.postListNamespaceDescriptors(descriptors);
- }
- }
- return descriptors;
+ @Override
+ public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
+ checkInitialized();
+ return listTableDescriptors(name, null, null, true);
}
@Override
@@ -2617,18 +2544,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
return procInfoList;
}
- @Override
- public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
- ensureNamespaceExists(name);
- return listTableDescriptors(name, null, null, true);
- }
-
- @Override
- public List<TableName> listTableNamesByNamespace(String name) throws IOException {
- ensureNamespaceExists(name);
- return listTableNames(name, null, true);
- }
-
/**
* Returns the list of table descriptors that match the specified request
*
@@ -2653,6 +2568,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
// request for all TableDescriptors
Collection<HTableDescriptor> htds;
if (namespace != null && namespace.length() > 0) {
+ // Do a check on the namespace existence. Will fail if does not exist.
+ this.clusterSchemaService.getNamespace(namespace);
htds = tableDescriptors.getByNamespace(namespace).values();
} else {
htds = tableDescriptors.getAll().values();
@@ -2696,46 +2613,17 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
*/
public List<TableName> listTableNames(final String namespace, final String regex,
final boolean includeSysTables) throws IOException {
- final List<HTableDescriptor> descriptors = new ArrayList<HTableDescriptor>();
-
- boolean bypass = false;
- if (cpHost != null) {
- bypass = cpHost.preGetTableNames(descriptors, regex);
- }
-
+ List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+ boolean bypass = cpHost != null? cpHost.preGetTableNames(htds, regex): false;
if (!bypass) {
- // get all descriptors
- Collection<HTableDescriptor> htds;
- if (namespace != null && namespace.length() > 0) {
- htds = tableDescriptors.getByNamespace(namespace).values();
- } else {
- htds = tableDescriptors.getAll().values();
- }
-
- for (HTableDescriptor htd: htds) {
- if (includeSysTables || !htd.getTableName().isSystemTable()) {
- descriptors.add(htd);
- }
- }
-
- // Retains only those matched by regular expression.
- if (regex != null) {
- filterTablesByRegex(descriptors, Pattern.compile(regex));
- }
-
- if (cpHost != null) {
- cpHost.postGetTableNames(descriptors, regex);
- }
- }
-
- List<TableName> result = new ArrayList<TableName>(descriptors.size());
- for (HTableDescriptor htd: descriptors) {
- result.add(htd.getTableName());
+ htds.addAll(listTableDescriptors(namespace, regex, null, includeSysTables));
+ if (cpHost != null) cpHost.postGetTableNames(htds, regex);
}
+ List<TableName> result = new ArrayList<TableName>(htds.size());
+ for (HTableDescriptor htd: htds) result.add(htd.getTableName());
return result;
}
-
/**
* Removes the table descriptors that don't match the pattern.
* @param descriptors list of table descriptors to filter
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index b269c3d..2528814 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceExistException;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.ServerLoad;
@@ -832,7 +833,7 @@ public class MasterRpcServices extends RSRpcServices
try {
return GetNamespaceDescriptorResponse.newBuilder()
.setNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(
- master.getNamespaceDescriptor(request.getNamespaceName())))
+ master.getNamespace(request.getNamespaceName())))
.build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -1120,7 +1121,7 @@ public class MasterRpcServices extends RSRpcServices
try {
ListNamespaceDescriptorsResponse.Builder response =
ListNamespaceDescriptorsResponse.newBuilder();
- for(NamespaceDescriptor ns: master.listNamespaceDescriptors()) {
+ for(NamespaceDescriptor ns: master.getNamespaces()) {
response.addNamespaceDescriptor(ProtobufUtil.toProtoNamespaceDescriptor(ns));
}
return response.build();
@@ -1305,10 +1306,9 @@ public class MasterRpcServices extends RSRpcServices
master.checkInitialized();
master.snapshotManager.checkSnapshotSupport();
- // ensure namespace exists
+ // Ensure namespace exists. Will throw exception if non-known NS.
TableName dstTable = TableName.valueOf(request.getSnapshot().getTable());
- master.getNamespaceDescriptor(dstTable.getNamespaceAsString());
-
+ master.getNamespace(dstTable.getNamespaceAsString());
SnapshotDescription reqSnapshot = request.getSnapshot();
master.snapshotManager.restoreSnapshot(reqSnapshot);
return RestoreSnapshotResponse.newBuilder().build();
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index af0e490..ec7db0c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -21,21 +21,20 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import com.google.protobuf.Service;
@@ -46,6 +45,11 @@ import com.google.protobuf.Service;
@InterfaceAudience.Private
public interface MasterServices extends Server {
/**
+ * @return Master's instance of {@link ClusterSchema}
+ */
+ ClusterSchema getClusterSchema();
+
+ /**
* @return Master's instance of the {@link AssignmentManager}
*/
AssignmentManager getAssignmentManager();
@@ -81,11 +85,6 @@ public interface MasterServices extends Server {
MasterCoprocessorHost getMasterCoprocessorHost();
/**
- * @return Master's instance of {@link TableNamespaceManager}
- */
- TableNamespaceManager getTableNamespaceManager();
-
- /**
* @return Master's instance of {@link MasterQuotaManager}
*/
MasterQuotaManager getMasterQuotaManager();
@@ -280,54 +279,6 @@ public interface MasterServices extends Server {
boolean isInitialized();
/**
- * Create a new namespace
- * @param descriptor descriptor which describes the new namespace
- * @param nonceGroup
- * @param nonce
- * @throws IOException
- */
- public void createNamespace(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException;
-
- /**
- * Create a new namespace synchronously.
- * @param descriptor descriptor which describes the new namespace
- * @param nonceGroup
- * @param nonce
- * @throws IOException
- */
- public void createNamespaceSync(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException;
-
- /**
- * Modify an existing namespace
- * @param descriptor descriptor which updates the existing namespace
- * @param nonceGroup
- * @param nonce
- * @throws IOException
- */
- public void modifyNamespace(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException;
-
- /**
- * Delete an existing namespace. Only empty namespaces (no tables) can be removed.
- * @param name namespace name
- * @param nonceGroup
- * @param nonce
- * @throws IOException
- */
- public void deleteNamespace(
- final String name,
- final long nonceGroup,
- final long nonce) throws IOException;
-
- /**
* Abort a procedure.
* @param procId ID of the procedure
* @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?
@@ -338,21 +289,6 @@ public interface MasterServices extends Server {
throws IOException;
/**
- * Get a namespace descriptor by name
- * @param name name of namespace descriptor
- * @return A descriptor
- * @throws IOException
- */
- public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException;
-
- /**
- * List available namespace descriptors
- * @return A descriptor
- * @throws IOException
- */
- public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException;
-
- /**
* List procedures
* @return procedure list
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java
new file mode 100644
index 0000000..0a849fc
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ProcedureFuture.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * Map Future Interface on to Procedure result processing.
+ */
+// Has no extra methods as of now beyond Future<ProcedureInfo>. Use #toString if you want to log
+// procId of procedure.
+// TODO: This should be in Procedure? Have it in master package for now. Lets out ProcedureInfo.
+// Implementation informed by HBaseAdmin#ProcedureFuture.
+@InterfaceAudience.Private
+class ProcedureFuture implements Future<ProcedureInfo> {
+ // Save exception so we can rethrow if called again. Same for result.
+ private ExecutionException exception = null;
+ private ProcedureInfo result = null;
+ private boolean done = false;
+ private boolean cancelled = false;
+ private final Long procId;
+ private final ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
+
+ ProcedureFuture(final ProcedureExecutor<MasterProcedureEnv> procedureExecutor,
+ final long procId) {
+ this.procedureExecutor = procedureExecutor;
+ this.procId = procId;
+ }
+
+ @Override
+ public String toString() {
+ return "procId=" + this.procId;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ if (!this.cancelled) {
+ this.cancelled = this.procedureExecutor.abort(this.procId, mayInterruptIfRunning);
+ }
+ return this.cancelled;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return this.cancelled;
+ }
+
+ @Override
+ public boolean isDone() {
+ return done;
+ }
+
+ /**
+ * This method is unsupported. We will throw an UnsupportedOperationException. Only the lazy
+ * would call this method because they can avoid thinking through implication of a Procedure that
+ * might never return so this is disallowed. Use {@link #get(long, TimeUnit)}.
+ */
+ @Override
+ public ProcedureInfo get() throws InterruptedException, ExecutionException {
+ // TODO: should we ever spin forever?
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcedureInfo get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ if (!this.done) {
+ // TODO: add this sort of facility to EnvironmentEdgeManager
+ long deadlineTs = EnvironmentEdgeManager.currentTime() + unit.toMillis(timeout);
+ try {
+ this.result = waitProcedureResult(procId, deadlineTs);
+ } catch (IOException e) {
+ this.exception = new ExecutionException(e);
+ }
+ this.done = true;
+ }
+ if (exception != null) {
+ throw exception;
+ }
+ return result;
+ }
+
+ /**
+ * @param procId
+ * @param deadlineTs
+ * @return A ProcedureInfo instance or null if procedure not found.
+ * @throws IOException
+ * @throws TimeoutException
+ * @throws InterruptedException
+ */
+ private ProcedureInfo waitProcedureResult(long procId, long deadlineTs)
+ throws IOException, TimeoutException, InterruptedException {
+ while (EnvironmentEdgeManager.currentTime() < deadlineTs) {
+ Pair<ProcedureInfo, Procedure> pair = this.procedureExecutor.getResultOrProcedure(procId);
+ if (pair.getFirst() != null) {
+ this.procedureExecutor.removeResult(procId);
+ return pair.getFirst();
+ } else {
+ if (pair.getSecond() == null) return null;
+ }
+ // TODO: Add a wait.
+ }
+ throw new TimeoutException("The procedure " + procId + " is still running");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
index bbeaf76..07822fd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java
@@ -27,17 +27,17 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZKNamespaceManager;
-import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.constraint.ConstraintException;
-import org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
@@ -56,10 +55,12 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import com.google.common.collect.Sets;
/**
- * This is a helper class used to manage the namespace
- * metadata that is stored in TableName.NAMESPACE_TABLE_NAME
- * It also mirrors updates to the ZK store by forwarding updates to
- * {@link org.apache.hadoop.hbase.ZKNamespaceManager}
+ * This is a helper class used internally to manage the namespace metadata that is stored in
+ * TableName.NAMESPACE_TABLE_NAME. It also mirrors updates to the ZK store by forwarding updates to
+ * {@link org.apache.hadoop.hbase.ZKNamespaceManager}.
+ *
+ * WARNING: Do not use. Go via the higher-level {@link ClusterSchema} API instead. This manager
+ * is likely to go aways anyways.
*/
@InterfaceAudience.Private
public class TableNamespaceManager {
@@ -90,7 +91,7 @@ public class TableNamespaceManager {
private long exclusiveLockTimeoutMs;
private long sharedLockTimeoutMs;
- public TableNamespaceManager(MasterServices masterServices) {
+ TableNamespaceManager(MasterServices masterServices) {
this.masterServices = masterServices;
this.conf = masterServices.getConfiguration();
@@ -113,7 +114,7 @@ public class TableNamespaceManager {
// Wait for the namespace table to be initialized.
long startTime = EnvironmentEdgeManager.currentTime();
int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
- while (!isTableAvailableAndInitialized(false)) {
+ while (!isTableAvailableAndInitialized()) {
if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) {
// We can't do anything if ns is not online.
throw new IOException("Timedout " + timeout + "ms waiting for namespace table to "
@@ -269,16 +270,29 @@ public class TableNamespaceManager {
}
/**
+ * Create Namespace in a blocking manner; don't return till success.
+ * Note, by-passes notifying coprocessors and name checks. Use for system namespaces only.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private void createNamespace(final NamespaceDescriptor namespaceDescriptor)
+ throws IOException {
+ ClusterSchema clusterSchema = this.masterServices.getClusterSchema();
+ clusterSchema.get(clusterSchema.
+ createNamespace(namespaceDescriptor, HConstants.NO_NONCE, HConstants.NO_NONCE));
+ }
+
+ /**
* This method checks if the namespace table is assigned and then
- * tries to create its HTable. If it was already created before, it also makes
+ * tries to create its Table reference. If it was already created before, it also makes
* sure that the connection isn't closed.
* @return true if the namespace table manager is ready to serve, false
* otherwise
* @throws IOException
*/
@SuppressWarnings("deprecation")
- public synchronized boolean isTableAvailableAndInitialized(
- final boolean createNamespaceAync) throws IOException {
+ public synchronized boolean isTableAvailableAndInitialized()
+ throws IOException {
// Did we already get a table? If so, still make sure it's available
if (isTableNamespaceManagerInitialized()) {
return true;
@@ -293,34 +307,10 @@ public class TableNamespaceManager {
zkNamespaceManager.start();
if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) {
- if (createNamespaceAync) {
- masterServices.getMasterProcedureExecutor().submitProcedure(
- new CreateNamespaceProcedure(
- masterServices.getMasterProcedureExecutor().getEnvironment(),
- NamespaceDescriptor.DEFAULT_NAMESPACE));
- initGoodSofar = false;
- }
- else {
- masterServices.createNamespaceSync(
- NamespaceDescriptor.DEFAULT_NAMESPACE,
- HConstants.NO_NONCE,
- HConstants.NO_NONCE);
- }
+ createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
}
if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) {
- if (createNamespaceAync) {
- masterServices.getMasterProcedureExecutor().submitProcedure(
- new CreateNamespaceProcedure(
- masterServices.getMasterProcedureExecutor().getEnvironment(),
- NamespaceDescriptor.SYSTEM_NAMESPACE));
- initGoodSofar = false;
- }
- else {
- masterServices.createNamespaceSync(
- NamespaceDescriptor.SYSTEM_NAMESPACE,
- HConstants.NO_NONCE,
- HConstants.NO_NONCE);
- }
+ createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
}
if (!initGoodSofar) {
@@ -410,4 +400,4 @@ public class TableNamespaceManager {
}
return maxRegions;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index 657bbfb..f934737 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -348,7 +348,7 @@ public class CreateNamespaceProcedure
}
private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) {
- return env.getMasterServices().getTableNamespaceManager();
+ return env.getMasterServices().getClusterSchema().getTableNamespaceManager();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index 5a42614..2f99167 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -383,7 +383,7 @@ public class DeleteNamespaceProcedure
}
private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) {
- return env.getMasterServices().getTableNamespaceManager();
+ return env.getMasterServices().getClusterSchema().getTableNamespaceManager();
}
/**
* The procedure could be restarted from a different machine. If the variable is null, we need to
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
index 30de252..0f8c172 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
@@ -266,8 +266,9 @@ public class ModifyNamespaceProcedure
}
private TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) {
- return env.getMasterServices().getTableNamespaceManager();
+ return env.getMasterServices().getClusterSchema().getTableNamespaceManager();
}
+
/**
* The procedure could be restarted from a different machine. If the variable is null, we need to
* retrieve it.
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
index f24f8c0..8035d32 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
@@ -65,7 +65,7 @@ class NamespaceStateManager {
/**
* Gets an instance of NamespaceTableAndRegionInfo associated with namespace.
- * @param The name of the namespace
+ * @param name The name of the namespace
* @return An instance of NamespaceTableAndRegionInfo.
*/
public NamespaceTableAndRegionInfo getState(String name) {
@@ -135,7 +135,7 @@ class NamespaceStateManager {
private NamespaceDescriptor getNamespaceDescriptor(String namespaceAsString) {
try {
- return this.master.getNamespaceDescriptor(namespaceAsString);
+ return this.master.getClusterSchema().getNamespace(namespaceAsString);
} catch (IOException e) {
LOG.error("Error while fetching namespace descriptor for namespace : " + namespaceAsString);
return null;
@@ -212,7 +212,7 @@ class NamespaceStateManager {
* Initialize namespace state cache by scanning meta table.
*/
private void initialize() throws IOException {
- List<NamespaceDescriptor> namespaces = this.master.listNamespaceDescriptors();
+ List<NamespaceDescriptor> namespaces = this.master.getClusterSchema().getNamespaces();
for (NamespaceDescriptor namespace : namespaces) {
addNamespace(namespace.getName());
List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 211fed5..1255fa4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -196,8 +196,7 @@ import sun.misc.SignalHandler;
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@SuppressWarnings("deprecation")
-public class HRegionServer extends HasThread implements
- RegionServerServices, LastSequenceId {
+public class HRegionServer extends HasThread implements RegionServerServices, LastSequenceId {
private static final Log LOG = LogFactory.getLog(HRegionServer.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
index c24d8a3..f9e2a16 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestNamespace.java
@@ -358,7 +358,7 @@ public class TestNamespace {
runWithExpectedException(new Callable<Void>() {
@Override
public Void call() throws Exception {
- admin.listTableDescriptorsByNamespace("non_existing_namespace");
+ admin.listTableDescriptorsByNamespace("non_existant_namespace");
return null;
}
}, NamespaceNotFoundException.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index 8e35bbf..785a006 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaMockingUtil;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.Server;
@@ -143,11 +142,10 @@ public class TestCatalogJanitor {
ServerName.valueOf("example.org,12345,6789"),
HRegionInfo.FIRST_META_REGIONINFO);
// Set hbase.rootdir into test dir.
- FileSystem fs = FileSystem.get(this.c);
+ FileSystem.get(this.c);
Path rootdir = FSUtils.getRootDir(this.c);
FSUtils.setRootDir(this.c, rootdir);
- AdminProtos.AdminService.BlockingInterface hri =
- Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
+ Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
}
@Override
@@ -402,48 +400,6 @@ public class TestCatalogJanitor {
}
@Override
- public void createNamespace(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void createNamespaceSync(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void modifyNamespace(
- final NamespaceDescriptor descriptor,
- final long nonceGroup,
- final long nonce) throws IOException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void deleteNamespace(
- final String name,
- final long nonceGroup,
- final long nonce) throws IOException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
throws IOException {
return false; //To change body of implemented methods use File | Settings | File Templates.
@@ -536,32 +492,29 @@ public class TestCatalogJanitor {
}
@Override
- public TableNamespaceManager getTableNamespaceManager() {
- return null;
- }
-
- @Override
public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
boolean forcible) throws IOException {
}
@Override
public boolean isInitialized() {
- // Auto-generated method stub
return false;
}
@Override
public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
- // Auto-generated method stub
return 0;
}
@Override
public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
- // Auto-generated method stub
return 0;
}
+
+ @Override
+ public ClusterSchema getClusterSchema() {
+ return null;
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/c5f3d17a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index 972834a..20b492c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -218,10 +218,6 @@ public class TestMasterNoCluster {
return null;
}
}
-
- @Override
- void initNamespace() {
- }
};
master.start();
@@ -294,10 +290,6 @@ public class TestMasterNoCluster {
return null;
}
}
-
- @Override
- void initNamespace() {
- }
};
master.start();
[02/29] hbase git commit: HBASE-14030 HBase Backup/Restore Phase 1
(Vladimir Rodionov)
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 9d9cee0..2ceeda5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -85,6 +85,9 @@ public class WALPlayer extends Configured implements Tool {
private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
+ public WALPlayer(){
+ }
+
protected WALPlayer(final Configuration c) {
super(c);
}
@@ -94,7 +97,7 @@ public class WALPlayer extends Configured implements Tool {
* This one can be used together with {@link KeyValueSortReducer}
*/
static class WALKeyValueMapper
- extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> {
+ extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> {
private byte[] table;
@Override
@@ -106,7 +109,9 @@ public class WALPlayer extends Configured implements Tool {
if (Bytes.equals(table, key.getTablename().getName())) {
for (Cell cell : value.getCells()) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
- if (WALEdit.isMetaEditFamily(kv)) continue;
+ if (WALEdit.isMetaEditFamily(kv)) {
+ continue;
+ }
context.write(new ImmutableBytesWritable(CellUtil.cloneRow(kv)), kv);
}
}
@@ -132,7 +137,7 @@ public class WALPlayer extends Configured implements Tool {
* a running HBase instance.
*/
protected static class WALMapper
- extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation> {
+ extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation> {
private Map<TableName, TableName> tables = new TreeMap<TableName, TableName>();
@Override
@@ -149,7 +154,9 @@ public class WALPlayer extends Configured implements Tool {
Cell lastCell = null;
for (Cell cell : value.getCells()) {
// filtering WAL meta entries
- if (WALEdit.isMetaEditFamily(cell)) continue;
+ if (WALEdit.isMetaEditFamily(cell)) {
+ continue;
+ }
// Allow a subclass filter out this cell.
if (filter(context, cell)) {
@@ -160,8 +167,12 @@ public class WALPlayer extends Configured implements Tool {
if (lastCell == null || lastCell.getTypeByte() != cell.getTypeByte()
|| !CellUtil.matchingRow(lastCell, cell)) {
// row or type changed, write out aggregate KVs.
- if (put != null) context.write(tableOut, put);
- if (del != null) context.write(tableOut, del);
+ if (put != null) {
+ context.write(tableOut, put);
+ }
+ if (del != null) {
+ context.write(tableOut, del);
+ }
if (CellUtil.isDelete(cell)) {
del = new Delete(CellUtil.cloneRow(cell));
} else {
@@ -177,8 +188,12 @@ public class WALPlayer extends Configured implements Tool {
lastCell = cell;
}
// write residual KVs
- if (put != null) context.write(tableOut, put);
- if (del != null) context.write(tableOut, del);
+ if (put != null) {
+ context.write(tableOut, put);
+ }
+ if (del != null) {
+ context.write(tableOut, del);
+ }
}
} catch (InterruptedException e) {
e.printStackTrace();
@@ -186,7 +201,8 @@ public class WALPlayer extends Configured implements Tool {
}
/**
- * @param cell
+ * Filter cell
+ * @param cell cell
* @return Return true if we are to emit this cell.
*/
protected boolean filter(Context context, final Cell cell) {
@@ -197,9 +213,7 @@ public class WALPlayer extends Configured implements Tool {
public void setup(Context context) throws IOException {
String[] tableMap = context.getConfiguration().getStrings(TABLE_MAP_KEY);
String[] tablesToUse = context.getConfiguration().getStrings(TABLES_KEY);
- if (tablesToUse == null && tableMap == null) {
- // Then user wants all tables.
- } else if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) {
+ if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) {
// this can only happen when WALMapper is used directly by a class other than WALPlayer
throw new IOException("No tables or incorrect table mapping specified.");
}
@@ -215,7 +229,9 @@ public class WALPlayer extends Configured implements Tool {
void setupTime(Configuration conf, String option) throws IOException {
String val = conf.get(option);
- if (null == val) return;
+ if (null == val) {
+ return;
+ }
long ms;
try {
// first try to parse in user friendly form
@@ -295,7 +311,8 @@ public class WALPlayer extends Configured implements Tool {
return job;
}
- /*
+ /**
+ * Print usage
* @param errorMsg Error message. Can be null.
*/
private void usage(final String errorMsg) {
@@ -305,7 +322,8 @@ public class WALPlayer extends Configured implements Tool {
System.err.println("Usage: " + NAME + " [options] <wal inputdir> <tables> [<tableMappings>]");
System.err.println("Read all WAL entries for <tables>.");
System.err.println("If no tables (\"\") are specific, all tables are imported.");
- System.err.println("(Careful, even -ROOT- and hbase:meta entries will be imported in that case.)");
+ System.err.println("(Careful, even -ROOT- and hbase:meta entries will be imported"+
+ " in that case.)");
System.err.println("Otherwise <tables> is a comma separated list of tables.\n");
System.err.println("The WAL entries can be mapped to new set of tables via <tableMapping>.");
System.err.println("<tableMapping> is a command separated list of targettables.");
@@ -318,10 +336,10 @@ public class WALPlayer extends Configured implements Tool {
System.err.println(" -D" + WALInputFormat.START_TIME_KEY + "=[date|ms]");
System.err.println(" -D" + WALInputFormat.END_TIME_KEY + "=[date|ms]");
System.err.println(" -D " + JOB_NAME_CONF_KEY
- + "=jobName - use the specified mapreduce job name for the wal player");
+ + "=jobName - use the specified mapreduce job name for the wal player");
System.err.println("For performance also consider the following options:\n"
- + " -Dmapreduce.map.speculative=false\n"
- + " -Dmapreduce.reduce.speculative=false");
+ + " -Dmapreduce.map.speculative=false\n"
+ + " -Dmapreduce.reduce.speculative=false");
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bdb19f4..5cd38b5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
+import org.apache.hadoop.hbase.backup.BackupManager;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
@@ -384,6 +385,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
Replication.decorateMasterConfiguration(this.conf);
+ BackupManager.decorateMasterConfiguration(this.conf);
// Hack! Maps DFSClient => Master for logs. HDFS made this
// config param for task trackers, but we can piggyback off of it.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
index 95c3ffe..b6e11ea 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
@@ -37,7 +37,7 @@ public abstract class RegionServerProcedureManager extends ProcedureManager {
* @param rss Region Server service interface
* @throws KeeperException
*/
- public abstract void initialize(RegionServerServices rss) throws KeeperException;
+ public abstract void initialize(RegionServerServices rss) throws IOException;
/**
* Start accepting procedure requests.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
index 0f4ea64..adb3604 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.procedure.flush.RegionServerFlushTableProcedureManager;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
-import org.apache.zookeeper.KeeperException;
/**
* Provides the globally barriered procedure framework and environment
@@ -39,7 +38,7 @@ public class RegionServerProcedureManagerHost extends
private static final Log LOG = LogFactory
.getLog(RegionServerProcedureManagerHost.class);
- public void initialize(RegionServerServices rss) throws KeeperException {
+ public void initialize(RegionServerServices rss) throws IOException {
for (RegionServerProcedureManager proc : procedures) {
LOG.debug("Procedure " + proc.getProcedureSignature() + " is initializing");
proc.initialize(rss);
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
index 085d642..3865ba9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
@@ -54,7 +54,7 @@ public class ZKProcedureCoordinatorRpcs implements ProcedureCoordinatorRpcs {
* @throws KeeperException if an unexpected zk error occurs
*/
public ZKProcedureCoordinatorRpcs(ZooKeeperWatcher watcher,
- String procedureClass, String coordName) throws KeeperException {
+ String procedureClass, String coordName) throws IOException {
this.watcher = watcher;
this.procedureType = procedureClass;
this.coordName = coordName;
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index 2e03a60..fff75a7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -68,49 +68,54 @@ public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
* @throws KeeperException if we can't reach zookeeper
*/
public ZKProcedureMemberRpcs(final ZooKeeperWatcher watcher, final String procType)
- throws KeeperException {
- this.zkController = new ZKProcedureUtil(watcher, procType) {
- @Override
- public void nodeCreated(String path) {
- if (!isInProcedurePath(path)) {
- return;
- }
+ throws IOException {
+ try {
+ this.zkController = new ZKProcedureUtil(watcher, procType) {
+ @Override
+ public void nodeCreated(String path) {
+ if (!isInProcedurePath(path)) {
+ return;
+ }
- LOG.info("Received created event:" + path);
- // if it is a simple start/end/abort then we just rewatch the node
- if (isAcquiredNode(path)) {
- waitForNewProcedures();
- return;
- } else if (isAbortNode(path)) {
- watchForAbortedProcedures();
- return;
+ LOG.info("Received created event:" + path);
+ // if it is a simple start/end/abort then we just rewatch the node
+ if (isAcquiredNode(path)) {
+ waitForNewProcedures();
+ return;
+ } else if (isAbortNode(path)) {
+ watchForAbortedProcedures();
+ return;
+ }
+ String parent = ZKUtil.getParent(path);
+ // if its the end barrier, the procedure can be completed
+ if (isReachedNode(parent)) {
+ receivedReachedGlobalBarrier(path);
+ return;
+ } else if (isAbortNode(parent)) {
+ abort(path);
+ return;
+ } else if (isAcquiredNode(parent)) {
+ startNewSubprocedure(path);
+ } else {
+ LOG.debug("Ignoring created notification for node:" + path);
+ }
}
- String parent = ZKUtil.getParent(path);
- // if its the end barrier, the procedure can be completed
- if (isReachedNode(parent)) {
- receivedReachedGlobalBarrier(path);
- return;
- } else if (isAbortNode(parent)) {
- abort(path);
- return;
- } else if (isAcquiredNode(parent)) {
- startNewSubprocedure(path);
- } else {
- LOG.debug("Ignoring created notification for node:" + path);
- }
- }
- @Override
- public void nodeChildrenChanged(String path) {
- if (path.equals(this.acquiredZnode)) {
- LOG.info("Received procedure start children changed event: " + path);
- waitForNewProcedures();
- } else if (path.equals(this.abortZnode)) {
- LOG.info("Received procedure abort children changed event: " + path);
- watchForAbortedProcedures();
+ @Override
+ public void nodeChildrenChanged(String path) {
+ if (path.equals(this.acquiredZnode)) {
+ LOG.info("Received procedure start children changed event: " + path);
+ waitForNewProcedures();
+ } else if (path.equals(this.abortZnode)) {
+ LOG.info("Received procedure abort children changed event: " + path);
+ watchForAbortedProcedures();
+ }
}
- }
- };
+ };
+ } catch (KeeperException e) {
+ // TODO Auto-generated catch block
+ throw new IOException(e);
+ }
}
public ZKProcedureUtil getZkController() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
index 1aa959c..bd65cc7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
@@ -317,7 +317,7 @@ public class RegionServerFlushTableProcedureManager extends RegionServerProcedur
* @throws KeeperException if the zookeeper cannot be reached
*/
@Override
- public void initialize(RegionServerServices rss) throws KeeperException {
+ public void initialize(RegionServerServices rss) throws IOException {
this.rss = rss;
ZooKeeperWatcher zkw = rss.getZooKeeper();
this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 211fed5..1cd54fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -807,8 +807,8 @@ public class HRegionServer extends HasThread implements
rspmHost = new RegionServerProcedureManagerHost();
rspmHost.loadProcedures(conf);
rspmHost.initialize(this);
- } catch (KeeperException e) {
- this.abort("Failed to reach zk cluster when creating procedure handler.", e);
+ } catch (IOException e) {
+ this.abort("Failed to reach coordination cluster when creating procedure handler.", e);
}
// register watcher for recovering regions
this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this);
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
index 537329a..e56dd28 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -390,7 +390,7 @@ public class RegionServerSnapshotManager extends RegionServerProcedureManager {
* @throws KeeperException if the zookeeper cluster cannot be reached
*/
@Override
- public void initialize(RegionServerServices rss) throws KeeperException {
+ public void initialize(RegionServerServices rss) throws IOException {
this.rss = rss;
ZooKeeperWatcher zkw = rss.getZooKeeper();
this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 9ae72e6..acde21e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -97,6 +97,8 @@ import com.lmax.disruptor.TimeoutException;
import com.lmax.disruptor.dsl.Disruptor;
import com.lmax.disruptor.dsl.ProducerType;
+
+
/**
* Implementation of {@link WAL} to go against {@link FileSystem}; i.e. keep WALs in HDFS.
* Only one WAL is ever being written at a time. When a WAL hits a configured maximum size,
@@ -359,7 +361,9 @@ public class FSHLog implements WAL {
public int compare(Path o1, Path o2) {
long t1 = getFileNumFromFileName(o1);
long t2 = getFileNumFromFileName(o2);
- if (t1 == t2) return 0;
+ if (t1 == t2) {
+ return 0;
+ }
return (t1 > t2) ? 1 : -1;
}
};
@@ -402,7 +406,7 @@ public class FSHLog implements WAL {
* @param root path for stored and archived wals
* @param logDir dir where wals are stored
* @param conf configuration to use
- * @throws IOException
+ * @throws IOException exception
*/
public FSHLog(final FileSystem fs, final Path root, final String logDir, final Configuration conf)
throws IOException {
@@ -410,7 +414,7 @@ public class FSHLog implements WAL {
}
/**
- * Create an edit log at the given <code>dir</code> location.
+ * Create an edit log at the given directory location.
*
* You should never have to load an existing log. If there is a log at
* startup, it should have already been processed and deleted by the time the
@@ -425,13 +429,13 @@ public class FSHLog implements WAL {
* be registered before we do anything else; e.g. the
* Constructor {@link #rollWriter()}.
* @param failIfWALExists If true IOException will be thrown if files related to this wal
- * already exist.
+ * already exist.
* @param prefix should always be hostname and port in distributed env and
- * it will be URL encoded before being used.
- * If prefix is null, "wal" will be used
+ * it will be URL encoded before being used.
+ * If prefix is null, "wal" will be used
* @param suffix will be url encoded. null is treated as empty. non-empty must start with
- * {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER}
- * @throws IOException
+ * {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER}
+ * @throws IOException exception
*/
public FSHLog(final FileSystem fs, final Path rootDir, final String logDir,
final String archiveDir, final Configuration conf,
@@ -593,7 +597,9 @@ public class FSHLog implements WAL {
@VisibleForTesting
OutputStream getOutputStream() {
FSDataOutputStream fsdos = this.hdfs_out;
- if (fsdos == null) return null;
+ if (fsdos == null) {
+ return null;
+ }
return fsdos.getWrappedStream();
}
@@ -628,7 +634,7 @@ public class FSHLog implements WAL {
/**
* Tell listeners about pre log roll.
- * @throws IOException
+ * @throws IOException exception
*/
private void tellListenersAboutPreLogRoll(final Path oldPath, final Path newPath)
throws IOException {
@@ -641,7 +647,7 @@ public class FSHLog implements WAL {
/**
* Tell listeners about post log roll.
- * @throws IOException
+ * @throws IOException exception
*/
private void tellListenersAboutPostLogRoll(final Path oldPath, final Path newPath)
throws IOException {
@@ -654,8 +660,7 @@ public class FSHLog implements WAL {
/**
* Run a sync after opening to set up the pipeline.
- * @param nextWriter
- * @param startTimeNanos
+ * @param nextWriter next writer
*/
private void preemptiveSync(final ProtobufLogWriter nextWriter) {
long startTimeNanos = System.nanoTime();
@@ -673,7 +678,9 @@ public class FSHLog implements WAL {
rollWriterLock.lock();
try {
// Return if nothing to flush.
- if (!force && (this.writer != null && this.numEntries.get() <= 0)) return null;
+ if (!force && (this.writer != null && this.numEntries.get() <= 0)) {
+ return null;
+ }
byte [][] regionsToFlush = null;
if (this.closed) {
LOG.debug("WAL closed. Skipping rolling of writer");
@@ -728,7 +735,7 @@ public class FSHLog implements WAL {
/**
* Archive old logs. A WAL is eligible for archiving if all its WALEdits have been flushed.
- * @throws IOException
+ * @throws IOException exception
*/
private void cleanOldLogs() throws IOException {
List<Path> logsToArchive = null;
@@ -738,9 +745,13 @@ public class FSHLog implements WAL {
Path log = e.getKey();
Map<byte[], Long> sequenceNums = e.getValue();
if (this.sequenceIdAccounting.areAllLower(sequenceNums)) {
- if (logsToArchive == null) logsToArchive = new ArrayList<Path>();
+ if (logsToArchive == null) {
+ logsToArchive = new ArrayList<Path>();
+ }
logsToArchive.add(log);
- if (LOG.isTraceEnabled()) LOG.trace("WAL file ready for archiving " + log);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("WAL file ready for archiving " + log);
+ }
}
}
if (logsToArchive != null) {
@@ -770,7 +781,9 @@ public class FSHLog implements WAL {
if (regions != null) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < regions.length; i++) {
- if (i > 0) sb.append(", ");
+ if (i > 0) {
+ sb.append(", ");
+ }
sb.append(Bytes.toStringBinary(regions[i]));
}
LOG.info("Too many WALs; count=" + logCount + ", max=" + this.maxLogs +
@@ -836,7 +849,9 @@ public class FSHLog implements WAL {
}
} catch (FailedSyncBeforeLogCloseException e) {
// If unflushed/unsynced entries on close, it is reason to abort.
- if (isUnflushedEntries()) throw e;
+ if (isUnflushedEntries()) {
+ throw e;
+ }
LOG.warn("Failed sync-before-close but no outstanding appends; closing WAL: " +
e.getMessage());
}
@@ -897,7 +912,9 @@ public class FSHLog implements WAL {
try {
blockOnSync(syncFuture);
} catch (IOException ioe) {
- if (LOG.isTraceEnabled()) LOG.trace("Stale sync exception", ioe);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Stale sync exception", ioe);
+ }
}
}
}
@@ -968,7 +985,15 @@ public class FSHLog implements WAL {
public Path getCurrentFileName() {
return computeFilename(this.filenum.get());
}
-
+
+ /**
+ * To support old API compatibility
+ * @return current file number (timestamp)
+ */
+ public long getFilenum() {
+ return filenum.get();
+ }
+
@Override
public String toString() {
return "FSHLog " + logFilePrefix + ":" + logFileSuffix + "(num " + filenum + ")";
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java
new file mode 100644
index 0000000..0360000
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.snapshot;
+
+import org.apache.hadoop.hbase.backup.BackupHandler;
+
+/* this class will be extended in future jira to support progress report */
+public class SnapshotCopy extends ExportSnapshot {
+ private BackupHandler backupHandler;
+ private String table;
+
+ public SnapshotCopy(BackupHandler backupHandler, String table) {
+ super();
+ this.backupHandler = backupHandler;
+ this.table = table;
+ }
+
+ public BackupHandler getBackupHandler() {
+ return this.backupHandler;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
index 027e7a2..dd4d337 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
@@ -209,13 +209,18 @@ public class DefaultWALProvider implements WALProvider {
@VisibleForTesting
public static long extractFileNumFromWAL(final WAL wal) {
final Path walName = ((FSHLog)wal).getCurrentFileName();
+ return extractFileNumFromWAL(walName);
+ }
+
+ @VisibleForTesting
+ public static long extractFileNumFromWAL(final Path walName) {
if (walName == null) {
throw new IllegalArgumentException("The WAL path couldn't be null");
}
final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
}
-
+
/**
* Pattern used to validate a WAL file name
* see {@link #validateWALFilename(String)} for description.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
new file mode 100644
index 0000000..bc0c848
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
@@ -0,0 +1,194 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * This class is only a base for other integration-level backup tests.
+ * Do not add tests here.
+ * TestBackupSmallTests is where tests that don't require bring machines up/down should go
+ * All other tests should have their own classes and extend this one
+ */
+public class TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestBackupBase.class);
+
+ protected static Configuration conf1;
+ protected static Configuration conf2;
+
+ protected static HBaseTestingUtility TEST_UTIL;
+ protected static HBaseTestingUtility TEST_UTIL2;
+
+ protected static TableName table1;
+ protected static TableName table2;
+ protected static TableName table3;
+ protected static TableName table4;
+
+ protected static String table1_restore = "table1_restore";
+ protected static String table2_restore = "table2_restore";
+ protected static String table3_restore = "table3_restore";
+ protected static String table4_restore = "table4_restore";
+
+ protected static final int NB_ROWS_IN_BATCH = 100;
+ protected static final byte[] qualName = Bytes.toBytes("q1");
+ protected static final byte[] famName = Bytes.toBytes("f");
+
+ protected static String BACKUP_ROOT_DIR = "/backupUT";
+ protected static String BACKUP_REMOTE_ROOT_DIR = "/backupUT";
+
+ protected static final String BACKUP_ZNODE = "/backup/hbase";
+ protected static final String BACKUP_SUCCEED_NODE = "complete";
+ protected static final String BACKUP_FAILED_NODE = "failed";
+
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+
+ TEST_UTIL = new HBaseTestingUtility();
+ TEST_UTIL.getConfiguration().set("hbase.procedure.regionserver.classes",
+ LogRollRegionServerProcedureManager.class.getName());
+ TEST_UTIL.getConfiguration().set("hbase.procedure.master.classes",
+ LogRollMasterProcedureManager.class.getName());
+ TEST_UTIL.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
+ TEST_UTIL.startMiniZKCluster();
+ MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
+
+ conf1 = TEST_UTIL.getConfiguration();
+ conf2 = HBaseConfiguration.create(conf1);
+ conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
+ TEST_UTIL2 = new HBaseTestingUtility(conf2);
+ TEST_UTIL2.setZkCluster(miniZK);
+ TEST_UTIL.startMiniCluster();
+ TEST_UTIL2.startMiniCluster();
+ conf1 = TEST_UTIL.getConfiguration();
+
+ TEST_UTIL.startMiniMapReduceCluster();
+ BACKUP_ROOT_DIR = TEST_UTIL.getConfiguration().get("fs.defaultFS") + "/backupUT";
+ LOG.info("ROOTDIR " + BACKUP_ROOT_DIR);
+ BACKUP_REMOTE_ROOT_DIR = TEST_UTIL2.getConfiguration().get("fs.defaultFS") + "/backupUT";
+ LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR);
+
+ BackupClient.setConf(conf1);
+ RestoreClient.setConf(conf1);
+ createTables();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
+ //zkw1.close();
+ TEST_UTIL2.shutdownMiniCluster();
+ TEST_UTIL.shutdownMiniCluster();
+ TEST_UTIL.shutdownMiniMapReduceCluster();
+ }
+
+ protected static void loadTable(HTable table) throws Exception {
+
+ Put p; // 100 + 1 row to t1_syncup
+ for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+ p = new Put(Bytes.toBytes("row" + i));
+ p.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ table.put(p);
+ }
+ }
+
+ protected static void createTables() throws Exception {
+
+ long tid = System.currentTimeMillis();
+ table1 = TableName.valueOf("test-" + tid);
+ HBaseAdmin ha = TEST_UTIL.getHBaseAdmin();
+ HTableDescriptor desc = new HTableDescriptor(table1);
+ HColumnDescriptor fam = new HColumnDescriptor(famName);
+ desc.addFamily(fam);
+ ha.createTable(desc);
+ Connection conn = ConnectionFactory.createConnection(conf1);
+ HTable table = (HTable) conn.getTable(table1);
+ loadTable(table);
+ table.close();
+ table2 = TableName.valueOf("test-" + tid + 1);
+ desc = new HTableDescriptor(table2);
+ desc.addFamily(fam);
+ ha.createTable(desc);
+ table = (HTable) conn.getTable(table2);
+ loadTable(table);
+ table.close();
+ table3 = TableName.valueOf("test-" + tid + 2);
+ table = TEST_UTIL.createTable(table3, famName);
+ table.close();
+ table4 = TableName.valueOf("test-" + tid + 3);
+ table = TEST_UTIL.createTable(table4, famName);
+ table.close();
+ ha.close();
+ conn.close();
+ }
+
+ protected boolean checkSucceeded(String backupId) throws IOException
+ {
+ BackupContext status = getBackupContext(backupId);
+ if(status == null) return false;
+ return status.getFlag() == BACKUPSTATUS.COMPLETE;
+ }
+
+ protected boolean checkFailed(String backupId) throws IOException
+ {
+ BackupContext status = getBackupContext(backupId);
+ if(status == null) return false;
+ return status.getFlag() == BACKUPSTATUS.FAILED;
+ }
+
+ private BackupContext getBackupContext(String backupId) throws IOException
+ {
+ Configuration conf = BackupClient.getConf();
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+ BackupContext status = table.readBackupStatus(backupId);
+ return status;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java
new file mode 100644
index 0000000..8be07bc
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestBackupBoundaryTests extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestBackupBoundaryTests.class);
+
+ /**
+ * Verify that full backup is created on a single empty table correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupSingleEmpty() throws Exception {
+
+ LOG.info("create full backup image on single table");
+
+ String backupId =
+ BackupClient.create("full", BACKUP_ROOT_DIR, table3.getNameAsString(), null);
+ LOG.info("Finished Backup");
+ assertTrue(checkSucceeded(backupId));
+ }
+
+ /**
+ * Verify that full backup is created on multiple empty tables correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupMultipleEmpty() throws Exception {
+ LOG.info("create full backup image on mulitple empty tables");
+ String tableset =
+ table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table4.getNameAsString();
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ assertTrue(checkSucceeded(backupId));
+
+ }
+
+ /**
+ * Verify that full backup fails on a single table that does not exist.
+ * @throws Exception
+ */
+ @Test(expected = RuntimeException.class)
+ public void testFullBackupSingleDNE() throws Exception {
+
+ LOG.info("test full backup fails on a single table that does not exist");
+ BackupClient.create("full", BACKUP_ROOT_DIR, "tabledne", null);
+ }
+
+ /**
+ * Verify that full backup fails on multiple tables that do not exist.
+ * @throws Exception
+ */
+ @Test(expected = RuntimeException.class)
+ public void testFullBackupMultipleDNE() throws Exception {
+
+ LOG.info("test full backup fails on multiple tables that do not exist");
+ BackupClient.create("full", BACKUP_ROOT_DIR, "table1dne,table2dne", null);
+ }
+
+ /**
+ * Verify that full backup fails on tableset containing real and fake tables.
+ * @throws Exception
+ */
+ @Test(expected = RuntimeException.class)
+ public void testFullBackupMixExistAndDNE() throws Exception {
+ LOG.info("create full backup fails on tableset containing real and fake table");
+ String tableset =
+ table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + "tabledne";
+ BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java
new file mode 100644
index 0000000..158479b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.DefaultWALProvider;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Iterables;
+
+@Category(LargeTests.class)
+public class TestBackupLogCleaner extends TestBackupBase {
+ private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class);
+
+ // implements all test cases in 1 test since incremental full backup/
+ // incremental backup has dependencies
+ @Test
+ public void testBackupLogCleaner() throws Exception {
+
+ // #1 - create full backup for all tables
+ LOG.info("create full backup image for all tables");
+ String tablesetFull =
+ table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table4.getNameAsString();
+
+ BackupSystemTable systemTable = BackupSystemTable.getTable(TEST_UTIL.getConfiguration());
+ // Verify that we have no backup sessions yet
+ assertFalse(systemTable.hasBackupSessions());
+
+ List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
+ List<String> swalFiles = convert(walFiles);
+ BackupLogCleaner cleaner = new BackupLogCleaner();
+ cleaner.setConf(TEST_UTIL.getConfiguration());
+
+ Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles);
+ // We can delete all files because we do not have yet recorded backup sessions
+ assertTrue(Iterables.size(deletable) == walFiles.size());
+
+ systemTable.addWALFiles(swalFiles, "backup");
+ String backupIdFull = BackupClient.create("full", BACKUP_ROOT_DIR, tablesetFull, null);
+ assertTrue(checkSucceeded(backupIdFull));
+ // Check one more time
+ deletable = cleaner.getDeletableFiles(walFiles);
+ // We can delete wal files because they were saved into hbase:backup table
+ int size = Iterables.size(deletable);
+ assertTrue(size == walFiles.size());
+
+ List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
+ LOG.debug("WAL list after full backup");
+ convert(newWalFiles);
+
+ // New list of wal files is greater than the previous one,
+ // because new wal per RS have been opened after full backup
+ assertTrue(walFiles.size() < newWalFiles.size());
+ // TODO : verify that result files are not walFiles collection
+ Connection conn = ConnectionFactory.createConnection(conf1);
+ // #2 - insert some data to table
+ HTable t1 = (HTable) conn.getTable(table1);
+ Put p1;
+ for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+ p1 = new Put(Bytes.toBytes("row-t1" + i));
+ p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t1.put(p1);
+ }
+
+ t1.close();
+
+ HTable t2 = (HTable) conn.getTable(table2);
+ Put p2;
+ for (int i = 0; i < 5; i++) {
+ p2 = new Put(Bytes.toBytes("row-t2" + i));
+ p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t2.put(p2);
+ }
+
+ t2.close();
+
+ // #3 - incremental backup for multiple tables
+ String tablesetIncMultiple =
+ table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString();
+
+ String backupIdIncMultiple =
+ BackupClient.create("incremental", BACKUP_ROOT_DIR, tablesetIncMultiple, null);
+ assertTrue(checkSucceeded(backupIdIncMultiple));
+ deletable = cleaner.getDeletableFiles(newWalFiles);
+
+ assertTrue(Iterables.size(deletable) == newWalFiles.size());
+
+ conn.close();
+
+ }
+
+ private List<String> convert(List<FileStatus> walFiles) {
+ List<String> result = new ArrayList<String>();
+ for (FileStatus fs : walFiles) {
+ LOG.debug("+++WAL: " + fs.getPath().toString());
+ result.add(fs.getPath().toString());
+ }
+ return result;
+ }
+
+ private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
+ Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME);
+ FileSystem fs = FileSystem.get(c);
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
+ List<FileStatus> logFiles = new ArrayList<FileStatus>();
+ while (it.hasNext()) {
+ LocatedFileStatus lfs = it.next();
+ if (lfs.isFile() && !DefaultWALProvider.isMetaFile(lfs.getPath())) {
+ logFiles.add(lfs);
+ LOG.info(lfs);
+ }
+ }
+ return logFiles;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java
new file mode 100644
index 0000000..7421707
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java
@@ -0,0 +1,341 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
+import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test cases for hbase:backup API
+ *
+ */
+@Category(MediumTests.class)
+public class TestBackupSystemTable {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ protected static Configuration conf = UTIL.getConfiguration();
+ protected static MiniHBaseCluster cluster;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ cluster = UTIL.startMiniCluster();
+
+ }
+
+ @Test
+ public void testUpdateReadDeleteBackupStatus() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+ BackupContext ctx = createBackupContext();
+ table.updateBackupStatus(ctx);
+ BackupContext readCtx = table.readBackupStatus(ctx.getBackupId());
+ assertTrue(compare(ctx, readCtx));
+
+ // try fake backup id
+ readCtx = table.readBackupStatus("fake");
+
+ assertNull(readCtx);
+ // delete backup context
+ table.deleteBackupStatus(ctx.getBackupId());
+ readCtx = table.readBackupStatus(ctx.getBackupId());
+ assertNull(readCtx);
+ cleanBackupTable();
+ }
+
+ @Test
+ public void testWriteReadBackupStartCode() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+ String code = "100";
+ table.writeBackupStartCode(code);
+ String readCode = table.readBackupStartCode();
+ assertEquals(code, readCode);
+ cleanBackupTable();
+ }
+
+ private void cleanBackupTable() throws IOException {
+ Admin admin = UTIL.getHBaseAdmin();
+ admin.disableTable(BackupSystemTable.getTableName());
+ admin.truncateTable(BackupSystemTable.getTableName(), true);
+ if (admin.isTableDisabled(BackupSystemTable.getTableName())) {
+ admin.enableTable(BackupSystemTable.getTableName());
+ }
+ }
+
+ @Test
+ public void testBackupHistory() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+ int n = 10;
+ List<BackupContext> list = createBackupContextList(n);
+
+ // Load data
+ for (BackupContext bc : list) {
+ // Make sure we set right status
+ bc.setFlag(BACKUPSTATUS.COMPLETE);
+ table.updateBackupStatus(bc);
+ }
+
+ // Reverse list for comparison
+ Collections.reverse(list);
+ ArrayList<BackupCompleteData> history = table.getBackupHistory();
+ assertTrue(history.size() == n);
+
+ for (int i = 0; i < n; i++) {
+ BackupContext ctx = list.get(i);
+ BackupCompleteData data = history.get(i);
+ assertTrue(compare(ctx, data));
+ }
+
+ cleanBackupTable();
+
+ }
+
+ @Test
+ public void testRegionServerLastLogRollResults() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+
+ String[] servers = new String[] { "server1", "server2", "server3" };
+ String[] timestamps = new String[] { "100", "102", "107" };
+
+ for (int i = 0; i < servers.length; i++) {
+ table.writeRegionServerLastLogRollResult(servers[i], timestamps[i]);
+ }
+
+ HashMap<String, String> result = table.readRegionServerLastLogRollResult();
+ assertTrue(servers.length == result.size());
+ Set<String> keys = result.keySet();
+ String[] keysAsArray = new String[keys.size()];
+ keys.toArray(keysAsArray);
+ Arrays.sort(keysAsArray);
+
+ for (int i = 0; i < keysAsArray.length; i++) {
+ assertEquals(keysAsArray[i], servers[i]);
+ String ts1 = timestamps[i];
+ String ts2 = result.get(keysAsArray[i]);
+ assertEquals(ts1, ts2);
+ }
+
+ cleanBackupTable();
+
+ }
+
+ @Test
+ public void testIncrementalBackupTableSet() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+
+ TreeSet<String> tables1 = new TreeSet<String>();
+
+ tables1.add("t1");
+ tables1.add("t2");
+ tables1.add("t3");
+
+ TreeSet<String> tables2 = new TreeSet<String>();
+
+ tables2.add("t3");
+ tables2.add("t4");
+ tables2.add("t5");
+
+ table.addIncrementalBackupTableSet(tables1);
+ TreeSet<String> res1 = (TreeSet<String>) table.getIncrementalBackupTableSet();
+ assertTrue(tables1.size() == res1.size());
+ Iterator<String> desc1 = tables1.descendingIterator();
+ Iterator<String> desc2 = res1.descendingIterator();
+ while (desc1.hasNext()) {
+ assertEquals(desc1.next(), desc2.next());
+ }
+
+ table.addIncrementalBackupTableSet(tables2);
+ TreeSet<String> res2 = (TreeSet<String>) table.getIncrementalBackupTableSet();
+ assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
+
+ tables1.addAll(tables2);
+
+ desc1 = tables1.descendingIterator();
+ desc2 = res2.descendingIterator();
+
+ while (desc1.hasNext()) {
+ assertEquals(desc1.next(), desc2.next());
+ }
+ cleanBackupTable();
+
+ }
+
+ @Test
+ public void testRegionServerLogTimestampMap() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+
+ TreeSet<String> tables = new TreeSet<String>();
+
+ tables.add("t1");
+ tables.add("t2");
+ tables.add("t3");
+
+ HashMap<String, String> rsTimestampMap = new HashMap<String, String>();
+
+ rsTimestampMap.put("rs1", "100");
+ rsTimestampMap.put("rs2", "101");
+ rsTimestampMap.put("rs3", "103");
+
+ table.writeRegionServerLogTimestamp(tables, rsTimestampMap);
+
+ HashMap<String, HashMap<String, String>> result = table.readLogTimestampMap();
+
+ assertTrue(tables.size() == result.size());
+
+ for (String t : tables) {
+ HashMap<String, String> rstm = result.get(t);
+ assertNotNull(rstm);
+ assertEquals(rstm.get("rs1"), "100");
+ assertEquals(rstm.get("rs2"), "101");
+ assertEquals(rstm.get("rs3"), "103");
+ }
+
+ Set<String> tables1 = new TreeSet<String>();
+
+ tables1.add("t3");
+ tables1.add("t4");
+ tables1.add("t5");
+
+ HashMap<String, String> rsTimestampMap1 = new HashMap<String, String>();
+
+ rsTimestampMap1.put("rs1", "200");
+ rsTimestampMap1.put("rs2", "201");
+ rsTimestampMap1.put("rs3", "203");
+
+ table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1);
+
+ result = table.readLogTimestampMap();
+
+ assertTrue(5 == result.size());
+
+ for (String t : tables) {
+ HashMap<String, String> rstm = result.get(t);
+ assertNotNull(rstm);
+ if (t.equals("t3") == false) {
+ assertEquals(rstm.get("rs1"), "100");
+ assertEquals(rstm.get("rs2"), "101");
+ assertEquals(rstm.get("rs3"), "103");
+ } else {
+ assertEquals(rstm.get("rs1"), "200");
+ assertEquals(rstm.get("rs2"), "201");
+ assertEquals(rstm.get("rs3"), "203");
+ }
+ }
+
+ for (String t : tables1) {
+ HashMap<String, String> rstm = result.get(t);
+ assertNotNull(rstm);
+ assertEquals(rstm.get("rs1"), "200");
+ assertEquals(rstm.get("rs2"), "201");
+ assertEquals(rstm.get("rs3"), "203");
+ }
+
+ cleanBackupTable();
+
+ }
+
+ @Test
+ public void testAddWALFiles() throws IOException {
+ BackupSystemTable table = BackupSystemTable.getTable(conf);
+ FileSystem fs = FileSystem.get(conf);
+ List<String> files =
+ Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1",
+ "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2",
+ "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3");
+ String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5";
+
+ table.addWALFiles(files, "backup");
+
+ assertTrue(table.checkWALFile(files.get(0)));
+ assertTrue(table.checkWALFile(files.get(1)));
+ assertTrue(table.checkWALFile(files.get(2)));
+ assertFalse(table.checkWALFile(newFile));
+
+ cleanBackupTable();
+ }
+
+ private boolean compare(BackupContext ctx, BackupCompleteData data) {
+
+ return ctx.getBackupId().equals(data.getBackupToken())
+ && ctx.getTargetRootDir().equals(data.getBackupRootPath())
+ && ctx.getType().equals(data.getType())
+ && ctx.getStartTs() == Long.parseLong(data.getStartTime())
+ && ctx.getEndTs() == Long.parseLong(data.getEndTime());
+
+ }
+
+ private boolean compare(BackupContext one, BackupContext two) {
+ return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
+ && one.getTargetRootDir().equals(two.getTargetRootDir())
+ && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs();
+ }
+
+ private BackupContext createBackupContext() {
+
+ BackupContext ctxt =
+ new BackupContext("backup_" + System.nanoTime(), "full", new String[] { "t1", "t2", "t3" },
+ "/hbase/backup", null);
+ ctxt.setStartTs(System.currentTimeMillis());
+ ctxt.setEndTs(System.currentTimeMillis() + 1);
+ return ctxt;
+ }
+
+ private List<BackupContext> createBackupContextList(int size) {
+ List<BackupContext> list = new ArrayList<BackupContext>();
+ for (int i = 0; i < size; i++) {
+ list.add(createBackupContext());
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ return list;
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ if (cluster != null) cluster.shutdown();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java
new file mode 100644
index 0000000..19caf37
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestFullBackup extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestFullBackup.class);
+
+ /**
+ * Verify that full backup is created on a single table with data correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupSingle() throws Exception {
+
+ LOG.info("test full backup on a single table with data");
+ String backupId =
+ BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+ }
+
+ /**
+ * Verify that full backup is created on multiple tables correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupMultiple() throws Exception {
+ LOG.info("create full backup image on multiple tables with data");
+ String tableset =
+ table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table2.getNameAsString();
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ assertTrue(checkSucceeded(backupId));
+
+ }
+
+ /**
+ * Verify that full backup is created on all tables correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupAll() throws Exception {
+ LOG.info("create full backup image on all tables");
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, null, null);
+ assertTrue(checkSucceeded(backupId));
+
+ }
+
+ /**
+ * Verify that full backup is created on a table correctly using a snapshot.
+ * @throws Exception
+ */
+ //@Test
+ //public void testFullBackupUsingSnapshot() throws Exception {
+ // HBaseAdmin hba = new HBaseAdmin(conf1);
+ //String snapshot = "snapshot";
+ //hba.snapshot(snapshot, table1);
+ //LOG.info("create full backup image on a table using snapshot");
+ //String backupId =
+ // BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(),
+ // snapshot);
+ // }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
new file mode 100644
index 0000000..1262c54
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestFullRestore extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestFullRestore.class);
+
+ /**
+ * Verify that a single table is restored to a new table
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingle() throws Exception {
+
+ LOG.info("test full restore on a single table empty table");
+ String backupId =
+ BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+
+ String[] tableset = new String[] { table1.getNameAsString() };
+ String[] tablemap = new String[] { table1_restore };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap,
+ false);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(TableName.valueOf(table1_restore)));
+ TEST_UTIL.deleteTable(TableName.valueOf(table1_restore));
+ hba.close();
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultiple() throws Exception {
+ LOG.info("create full backup image on multiple tables");
+ String tableset =
+ table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString();
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ assertTrue(checkSucceeded(backupId));
+
+ String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() };
+ String[] tablemap = new String[] { table2_restore, table3_restore };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false,
+ restore_tableset, tablemap, false);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(TableName.valueOf(table2_restore)));
+ assertTrue(hba.tableExists(TableName.valueOf(table3_restore)));
+ TEST_UTIL.deleteTable(TableName.valueOf(table2_restore));
+ TEST_UTIL.deleteTable(TableName.valueOf(table3_restore));
+ hba.close();
+ }
+
+ /**
+ * Verify that a single table is restored using overwrite
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingleOverwrite() throws Exception {
+
+ LOG.info("test full restore on a single table empty table");
+ String backupId =
+ BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+
+ String[] tableset = new String[] { table1.getNameAsString() };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, null,
+ true);
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables using overwrite.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleOverwrite() throws Exception {
+ LOG.info("create full backup image on multiple tables");
+ String tableset =
+ table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString();
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ assertTrue(checkSucceeded(backupId));
+
+ String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false,
+ false, restore_tableset, null, true);
+ }
+
+ /**
+ * Verify that restore fails on a single table that does not exist.
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testFullRestoreSingleDNE() throws Exception {
+
+ LOG.info("test restore fails on a single table that does not exist");
+ String backupId =
+ BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+
+ String[] tableset = new String[] { "faketable" };
+ String[] tablemap = new String[] { table1_restore };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap,
+ false);
+ }
+
+ /**
+ * Verify that restore fails on multiple tables that do not exist.
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testFullRestoreMultipleDNE() throws Exception {
+
+ LOG.info("test restore fails on multiple tables that do not exist");
+ String tableset =
+ table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString();
+ String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
+ assertTrue(checkSucceeded(backupId));
+
+ String[] restore_tableset = new String[] { "faketable1", "faketable2" };
+ String[] tablemap = new String[] { table2_restore, table3_restore };
+ Path path = new Path(BACKUP_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false,
+ false, restore_tableset, tablemap, false);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
new file mode 100644
index 0000000..5437b84
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.backup.HBackupFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestIncrementalBackup extends TestBackupBase {
+ private static final Log LOG = LogFactory.getLog(TestIncrementalBackup.class);
+ //implement all testcases in 1 test since incremental backup/restore has dependencies
+ @Test
+ public void TestIncBackupRestore() throws Exception {
+ HBackupFileSystem hbfs;
+
+ // #1 - create full backup for all tables
+ LOG.info("create full backup image for all tables");
+ String tablesetFull =
+ table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table4.getNameAsString();
+
+ String backupIdFull =
+ BackupClient.create("full", BACKUP_ROOT_DIR, tablesetFull, null);
+ assertTrue(checkSucceeded(backupIdFull));
+
+ Connection conn = ConnectionFactory.createConnection(conf1);
+ // #2 - insert some data to table
+ HTable t1 = (HTable) conn.getTable(table1);
+ Put p1;
+ for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+ p1 = new Put(Bytes.toBytes("row-t1" + i));
+ p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t1.put(p1);
+ }
+
+ Assert.assertThat(TEST_UTIL.countRows(t1), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2));
+ t1.close();
+
+ HTable t2 = (HTable) conn.getTable(table2);
+ Put p2;
+ for (int i = 0; i < 5; i++) {
+ p2 = new Put(Bytes.toBytes("row-t2" + i));
+ p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t2.put(p2);
+ }
+
+ Assert.assertThat(TEST_UTIL.countRows(t2), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5));
+ t2.close();
+
+ // #3 - incremental backup for multiple tables
+ String tablesetIncMultiple =
+ table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
+ + table3.getNameAsString();
+
+ String backupIdIncMultiple = BackupClient.create("incremental", BACKUP_ROOT_DIR,
+ tablesetIncMultiple, null);
+ assertTrue(checkSucceeded(backupIdIncMultiple));
+
+
+ // #4 - restore full backup for all tables, without overwrite
+ String[] tablesRestoreFull =
+ new String[] { table1.getNameAsString(), table2.getNameAsString(),
+ table3.getNameAsString(), table4.getNameAsString() };
+
+ String[] tablesMapFull =
+ new String[] { table1_restore, table2_restore, table3_restore, table4_restore };
+
+ hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdFull);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdFull, false, false,
+ tablesRestoreFull,
+ tablesMapFull, false);
+
+ // #5.1 - check tables for full restore
+ HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hAdmin.tableExists(TableName.valueOf(table1_restore)));
+ assertTrue(hAdmin.tableExists(TableName.valueOf(table2_restore)));
+ assertTrue(hAdmin.tableExists(TableName.valueOf(table3_restore)));
+ assertTrue(hAdmin.tableExists(TableName.valueOf(table4_restore)));
+
+ hAdmin.close();
+
+ // #5.2 - checking row count of tables for full restore
+ HTable hTable = (HTable) conn.getTable(TableName.valueOf(table1_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table2_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table3_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table4_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
+ hTable.close();
+
+ // #6 - restore incremental backup for multiple tables, with overwrite
+ String[] tablesRestoreIncMultiple =
+ new String[]
+ { table1.getNameAsString(), table2.getNameAsString(), table3.getNameAsString() };
+ String[] tablesMapIncMultiple =
+ new String[] { table1_restore, table2_restore, table3_restore };
+ hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdIncMultiple, false, false,
+ tablesRestoreIncMultiple, tablesMapIncMultiple, true);
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table1_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table2_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table3_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
+ hTable.close();
+
+ // #7 - incremental backup for single, empty table
+
+ String tablesetIncEmpty = table4.getNameAsString();
+ String backupIdIncEmpty =
+ BackupClient.create("incremental", BACKUP_ROOT_DIR, tablesetIncEmpty, null);
+ assertTrue(checkSucceeded(backupIdIncEmpty));
+
+
+ // #8 - restore incremental backup for single empty table, with overwrite
+ String[] tablesRestoreIncEmpty = new String[] { table4.getNameAsString() };
+ String[] tablesMapIncEmpty = new String[] { table4_restore };
+ hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncEmpty);
+ RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdIncEmpty, false, false,
+ tablesRestoreIncEmpty,
+ tablesMapIncEmpty, true);
+
+ hTable = (HTable) conn.getTable(TableName.valueOf(table4_restore));
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
+ hTable.close();
+ conn.close();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
new file mode 100644
index 0000000..03822a2
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestRemoteBackup extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestRemoteBackup.class);
+
+ /**
+ * Verify that a remote full backup is created on a single table with data correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupRemote() throws Exception {
+
+ LOG.info("test remote full backup on a single table");
+
+ // String rootdir = TEST_UTIL2.getDefaultRootDirPath() + BACKUP_ROOT_DIR;
+ // LOG.info("ROOTDIR " + rootdir);
+ String backupId =
+ BackupClient.create("full", BACKUP_REMOTE_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
new file mode 100644
index 0000000..e1315c5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestRemoteRestore extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestRemoteRestore.class);
+
+ /**
+ * Verify that a remote restore on a single table is successful.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreRemote() throws Exception {
+
+ LOG.info("test remote full backup on a single table");
+ String backupId =
+ BackupClient.create("full", BACKUP_REMOTE_ROOT_DIR, table1.getNameAsString(), null);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+ String[] tableset = new String[] { table1.getNameAsString() };
+ String[] tablemap = new String[] { table1_restore };
+ Path path = new Path(BACKUP_REMOTE_ROOT_DIR);
+ HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
+ RestoreClient.restore_stage1(hbfs, BACKUP_REMOTE_ROOT_DIR, backupId, false, false, tableset,
+ tablemap, false);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(TableName.valueOf(table1_restore)));
+ TEST_UTIL.deleteTable(TableName.valueOf(table1_restore));
+ hba.close();
+ }
+
+}
\ No newline at end of file
[15/29] hbase git commit: HBASE-15023 Reenable TestShell and
TestStochasticLoadBalancer
Posted by sy...@apache.org.
HBASE-15023 Reenable TestShell and TestStochasticLoadBalancer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2fba25b6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2fba25b6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2fba25b6
Branch: refs/heads/hbase-12439
Commit: 2fba25b66aabcee199719114eebafdcaf6cc2942
Parents: a999c2a
Author: stack <st...@apache.org>
Authored: Mon Dec 28 13:27:17 2015 -0800
Committer: stack <st...@apache.org>
Committed: Mon Dec 28 13:27:17 2015 -0800
----------------------------------------------------------------------
.../balancer/TestStochasticLoadBalancer.java | 532 +++++++++++++++++++
.../apache/hadoop/hbase/client/TestShell.java | 39 ++
2 files changed, 571 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2fba25b6/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
new file mode 100644
index 0000000..7abbeb4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -0,0 +1,532 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.balancer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Queue;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.RegionLoad;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.master.RackManager;
+import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
+import org.apache.hadoop.hbase.testclassification.FlakeyTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({FlakeyTests.class, MediumTests.class})
+public class TestStochasticLoadBalancer extends BalancerTestBase {
+ public static final String REGION_KEY = "testRegion";
+ private static final Log LOG = LogFactory.getLog(TestStochasticLoadBalancer.class);
+
+ @Test
+ public void testKeepRegionLoad() throws Exception {
+
+ ServerName sn = ServerName.valueOf("test:8080", 100);
+ int numClusterStatusToAdd = 20000;
+ for (int i = 0; i < numClusterStatusToAdd; i++) {
+ ServerLoad sl = mock(ServerLoad.class);
+
+ RegionLoad rl = mock(RegionLoad.class);
+ when(rl.getStores()).thenReturn(i);
+
+ Map<byte[], RegionLoad> regionLoadMap =
+ new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
+ regionLoadMap.put(Bytes.toBytes(REGION_KEY), rl);
+ when(sl.getRegionsLoad()).thenReturn(regionLoadMap);
+
+ ClusterStatus clusterStatus = mock(ClusterStatus.class);
+ when(clusterStatus.getServers()).thenReturn(Arrays.asList(sn));
+ when(clusterStatus.getLoad(sn)).thenReturn(sl);
+
+ loadBalancer.setClusterStatus(clusterStatus);
+ }
+ assertTrue(loadBalancer.loads.get(REGION_KEY) != null);
+ assertTrue(loadBalancer.loads.get(REGION_KEY).size() == 15);
+
+ Queue<RegionLoad> loads = loadBalancer.loads.get(REGION_KEY);
+ int i = 0;
+ while(loads.size() > 0) {
+ RegionLoad rl = loads.remove();
+ assertEquals(i + (numClusterStatusToAdd - 15), rl.getStores());
+ i ++;
+ }
+ }
+
+ /**
+ * Test the load balancing algorithm.
+ *
+ * Invariant is that all servers should be hosting either floor(average) or
+ * ceiling(average)
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testBalanceCluster() throws Exception {
+
+ for (int[] mockCluster : clusterStateMocks) {
+ Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
+ List<ServerAndLoad> list = convertToList(servers);
+ LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
+ List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
+ List<ServerAndLoad> balancedCluster = reconcile(list, plans, servers);
+ LOG.info("Mock Balance : " + printMock(balancedCluster));
+ assertClusterAsBalanced(balancedCluster);
+ List<RegionPlan> secondPlans = loadBalancer.balanceCluster(servers);
+ assertNull(secondPlans);
+ for (Map.Entry<ServerName, List<HRegionInfo>> entry : servers.entrySet()) {
+ returnRegions(entry.getValue());
+ returnServer(entry.getKey());
+ }
+ }
+
+ }
+
+ @Test
+ public void testMoveCost() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.CostFunction
+ costFunction = new StochasticLoadBalancer.MoveCostFunction(conf);
+ for (int[] mockCluster : clusterStateMocks) {
+ BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
+ costFunction.init(cluster);
+ double cost = costFunction.cost();
+ assertEquals(0.0f, cost, 0.001);
+
+ // cluster region number is smaller than maxMoves=600
+ cluster.setNumRegions(200);
+ cluster.setNumMovedRegions(10);
+ cost = costFunction.cost();
+ assertEquals(0.05f, cost, 0.001);
+ cluster.setNumMovedRegions(100);
+ cost = costFunction.cost();
+ assertEquals(0.5f, cost, 0.001);
+ cluster.setNumMovedRegions(200);
+ cost = costFunction.cost();
+ assertEquals(1.0f, cost, 0.001);
+
+
+ // cluster region number is bigger than maxMoves=2500
+ cluster.setNumRegions(10000);
+ cluster.setNumMovedRegions(250);
+ cost = costFunction.cost();
+ assertEquals(0.1f, cost, 0.001);
+ cluster.setNumMovedRegions(1250);
+ cost = costFunction.cost();
+ assertEquals(0.5f, cost, 0.001);
+ cluster.setNumMovedRegions(2500);
+ cost = costFunction.cost();
+ assertEquals(1.0f, cost, 0.01);
+ }
+ }
+
+ @Test
+ public void testSkewCost() {
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.CostFunction
+ costFunction = new StochasticLoadBalancer.RegionCountSkewCostFunction(conf);
+ for (int[] mockCluster : clusterStateMocks) {
+ costFunction.init(mockCluster(mockCluster));
+ double cost = costFunction.cost();
+ assertTrue(cost >= 0);
+ assertTrue(cost <= 1.01);
+ }
+
+ costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
+ assertEquals(0,costFunction.cost(), 0.01);
+ costFunction.init(mockCluster(new int[]{0, 0, 0, 1, 1}));
+ assertEquals(0, costFunction.cost(), 0.01);
+ costFunction.init(mockCluster(new int[]{0, 0, 1, 1, 1}));
+ assertEquals(0, costFunction.cost(), 0.01);
+ costFunction.init(mockCluster(new int[]{0, 1, 1, 1, 1}));
+ assertEquals(0, costFunction.cost(), 0.01);
+ costFunction.init(mockCluster(new int[]{1, 1, 1, 1, 1}));
+ assertEquals(0, costFunction.cost(), 0.01);
+ costFunction.init(mockCluster(new int[]{10000, 0, 0, 0, 0}));
+ assertEquals(1, costFunction.cost(), 0.01);
+ }
+
+ @Test
+ public void testTableSkewCost() {
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.CostFunction
+ costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
+ for (int[] mockCluster : clusterStateMocks) {
+ BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
+ costFunction.init(cluster);
+ double cost = costFunction.cost();
+ assertTrue(cost >= 0);
+ assertTrue(cost <= 1.01);
+ }
+ }
+
+ @Test
+ public void testCostFromArray() {
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.CostFromRegionLoadFunction
+ costFunction = new StochasticLoadBalancer.MemstoreSizeCostFunction(conf);
+ costFunction.init(mockCluster(new int[]{0, 0, 0, 0, 1}));
+
+ double[] statOne = new double[100];
+ for (int i =0; i < 100; i++) {
+ statOne[i] = 10;
+ }
+ assertEquals(0, costFunction.costFromArray(statOne), 0.01);
+
+ double[] statTwo= new double[101];
+ for (int i =0; i < 100; i++) {
+ statTwo[i] = 0;
+ }
+ statTwo[100] = 100;
+ assertEquals(1, costFunction.costFromArray(statTwo), 0.01);
+
+ double[] statThree = new double[200];
+ for (int i =0; i < 100; i++) {
+ statThree[i] = (0);
+ statThree[i+100] = 100;
+ }
+ assertEquals(0.5, costFunction.costFromArray(statThree), 0.01);
+ }
+
+ @Test(timeout = 60000)
+ public void testLosingRs() throws Exception {
+ int numNodes = 3;
+ int numRegions = 20;
+ int numRegionsPerServer = 3; //all servers except one
+ int replication = 1;
+ int numTables = 2;
+
+ Map<ServerName, List<HRegionInfo>> serverMap =
+ createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
+ List<ServerAndLoad> list = convertToList(serverMap);
+
+
+ List<RegionPlan> plans = loadBalancer.balanceCluster(serverMap);
+ assertNotNull(plans);
+
+ // Apply the plan to the mock cluster.
+ List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
+
+ assertClusterAsBalanced(balancedCluster);
+
+ ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];
+
+ ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);
+
+ serverMap.put(deadSn, new ArrayList<HRegionInfo>(0));
+
+ plans = loadBalancer.balanceCluster(serverMap);
+ assertNull(plans);
+ }
+
+ @Test
+ public void testReplicaCost() {
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.CostFunction
+ costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
+ for (int[] mockCluster : clusterStateMocks) {
+ BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
+ costFunction.init(cluster);
+ double cost = costFunction.cost();
+ assertTrue(cost >= 0);
+ assertTrue(cost <= 1.01);
+ }
+ }
+
+ @Test
+ public void testReplicaCostForReplicas() {
+ Configuration conf = HBaseConfiguration.create();
+ StochasticLoadBalancer.CostFunction
+ costFunction = new StochasticLoadBalancer.RegionReplicaHostCostFunction(conf);
+
+ int [] servers = new int[] {3,3,3,3,3};
+ TreeMap<ServerName, List<HRegionInfo>> clusterState = mockClusterServers(servers);
+
+ BaseLoadBalancer.Cluster cluster;
+
+ cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
+ costFunction.init(cluster);
+ double costWithoutReplicas = costFunction.cost();
+ assertEquals(0, costWithoutReplicas, 0);
+
+ // replicate the region from first server to the last server
+ HRegionInfo replica1 = RegionReplicaUtil.getRegionInfoForReplica(
+ clusterState.firstEntry().getValue().get(0),1);
+ clusterState.lastEntry().getValue().add(replica1);
+
+ cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
+ costFunction.init(cluster);
+ double costWith1ReplicaDifferentServer = costFunction.cost();
+
+ assertEquals(0, costWith1ReplicaDifferentServer, 0);
+
+ // add a third replica to the last server
+ HRegionInfo replica2 = RegionReplicaUtil.getRegionInfoForReplica(replica1, 2);
+ clusterState.lastEntry().getValue().add(replica2);
+
+ cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
+ costFunction.init(cluster);
+ double costWith1ReplicaSameServer = costFunction.cost();
+
+ assertTrue(costWith1ReplicaDifferentServer < costWith1ReplicaSameServer);
+
+ // test with replication = 4 for following:
+
+ HRegionInfo replica3;
+ Iterator<Entry<ServerName, List<HRegionInfo>>> it;
+ Entry<ServerName, List<HRegionInfo>> entry;
+
+ clusterState = mockClusterServers(servers);
+ it = clusterState.entrySet().iterator();
+ entry = it.next(); //first server
+ HRegionInfo hri = entry.getValue().get(0);
+ replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
+ replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
+ replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
+ entry.getValue().add(replica1);
+ entry.getValue().add(replica2);
+ it.next().getValue().add(replica3); //2nd server
+
+ cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
+ costFunction.init(cluster);
+ double costWith3ReplicasSameServer = costFunction.cost();
+
+ clusterState = mockClusterServers(servers);
+ hri = clusterState.firstEntry().getValue().get(0);
+ replica1 = RegionReplicaUtil.getRegionInfoForReplica(hri, 1);
+ replica2 = RegionReplicaUtil.getRegionInfoForReplica(hri, 2);
+ replica3 = RegionReplicaUtil.getRegionInfoForReplica(hri, 3);
+
+ clusterState.firstEntry().getValue().add(replica1);
+ clusterState.lastEntry().getValue().add(replica2);
+ clusterState.lastEntry().getValue().add(replica3);
+
+ cluster = new BaseLoadBalancer.Cluster(clusterState, null, null, null);
+ costFunction.init(cluster);
+ double costWith2ReplicasOnTwoServers = costFunction.cost();
+
+ assertTrue(costWith2ReplicasOnTwoServers < costWith3ReplicasSameServer);
+ }
+
+ @Test
+ public void testNeedsBalanceForColocatedReplicas() {
+ // check for the case where there are two hosts and with one rack, and where
+ // both the replicas are hosted on the same server
+ List<HRegionInfo> regions = randomRegions(1);
+ ServerName s1 = ServerName.valueOf("host1", 1000, 11111);
+ ServerName s2 = ServerName.valueOf("host11", 1000, 11111);
+ Map<ServerName, List<HRegionInfo>> map = new HashMap<ServerName, List<HRegionInfo>>();
+ map.put(s1, regions);
+ regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
+ // until the step above s1 holds two replicas of a region
+ regions = randomRegions(1);
+ map.put(s2, regions);
+ assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, null)));
+ // check for the case where there are two hosts on the same rack and there are two racks
+ // and both the replicas are on the same rack
+ map.clear();
+ regions = randomRegions(1);
+ List<HRegionInfo> regionsOnS2 = new ArrayList<HRegionInfo>(1);
+ regionsOnS2.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
+ map.put(s1, regions);
+ map.put(s2, regionsOnS2);
+ // add another server so that the cluster has some host on another rack
+ map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1));
+ assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null,
+ new ForTestRackManagerOne())));
+ }
+
+ @Test (timeout = 60000)
+ public void testSmallCluster() {
+ int numNodes = 10;
+ int numRegions = 1000;
+ int numRegionsPerServer = 40; //all servers except one
+ int replication = 1;
+ int numTables = 10;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+
+ @Test (timeout = 60000)
+ public void testSmallCluster2() {
+ int numNodes = 20;
+ int numRegions = 2000;
+ int numRegionsPerServer = 40; //all servers except one
+ int replication = 1;
+ int numTables = 10;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+
+ @Test (timeout = 60000)
+ public void testSmallCluster3() {
+ int numNodes = 20;
+ int numRegions = 2000;
+ int numRegionsPerServer = 1; // all servers except one
+ int replication = 1;
+ int numTables = 10;
+ /* fails because of max moves */
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, false, false);
+ }
+
+ @Test (timeout = 800000)
+ public void testMidCluster() {
+ int numNodes = 100;
+ int numRegions = 10000;
+ int numRegionsPerServer = 60; // all servers except one
+ int replication = 1;
+ int numTables = 40;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+
+ @Test (timeout = 800000)
+ public void testMidCluster2() {
+ int numNodes = 200;
+ int numRegions = 100000;
+ int numRegionsPerServer = 40; // all servers except one
+ int replication = 1;
+ int numTables = 400;
+ testWithCluster(numNodes,
+ numRegions,
+ numRegionsPerServer,
+ replication,
+ numTables,
+ false, /* num large num regions means may not always get to best balance with one run */
+ false);
+ }
+
+
+ @Test (timeout = 800000)
+ public void testMidCluster3() {
+ int numNodes = 100;
+ int numRegions = 2000;
+ int numRegionsPerServer = 9; // all servers except one
+ int replication = 1;
+ int numTables = 110;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ // TODO(eclark): Make sure that the tables are well distributed.
+ }
+
+ @Test
+ public void testLargeCluster() {
+ int numNodes = 1000;
+ int numRegions = 100000; //100 regions per RS
+ int numRegionsPerServer = 80; //all servers except one
+ int numTables = 100;
+ int replication = 1;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+
+ @Test (timeout = 800000)
+ public void testRegionReplicasOnSmallCluster() {
+ int numNodes = 10;
+ int numRegions = 1000;
+ int replication = 3; // 3 replicas per region
+ int numRegionsPerServer = 80; //all regions are mostly balanced
+ int numTables = 10;
+ testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true, true);
+ }
+
+ @Test (timeout = 800000)
+ public void testRegionReplicationOnMidClusterSameHosts() {
+ conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
+ conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
+ conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
+ loadBalancer.setConf(conf);
+ int numHosts = 100;
+ int numRegions = 100 * 100;
+ int replication = 3; // 3 replicas per region
+ int numRegionsPerServer = 5;
+ int numTables = 10;
+ Map<ServerName, List<HRegionInfo>> serverMap =
+ createServerMap(numHosts, numRegions, numRegionsPerServer, replication, numTables);
+ int numNodesPerHost = 4;
+
+ // create a new map with 4 RS per host.
+ Map<ServerName, List<HRegionInfo>> newServerMap = new TreeMap<ServerName, List<HRegionInfo>>(serverMap);
+ for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverMap.entrySet()) {
+ for (int i=1; i < numNodesPerHost; i++) {
+ ServerName s1 = entry.getKey();
+ ServerName s2 = ServerName.valueOf(s1.getHostname(), s1.getPort() + i, 1); // create an RS for the same host
+ newServerMap.put(s2, new ArrayList<HRegionInfo>());
+ }
+ }
+
+ testWithCluster(newServerMap, null, true, true);
+ }
+
+ private static class ForTestRackManager extends RackManager {
+ int numRacks;
+ public ForTestRackManager(int numRacks) {
+ this.numRacks = numRacks;
+ }
+ @Override
+ public String getRack(ServerName server) {
+ return "rack_" + (server.hashCode() % numRacks);
+ }
+ }
+
+ private static class ForTestRackManagerOne extends RackManager {
+ @Override
+ public String getRack(ServerName server) {
+ return server.getHostname().endsWith("1") ? "rack1" : "rack2";
+ }
+ }
+
+ @Test (timeout = 800000)
+ public void testRegionReplicationOnMidClusterWithRacks() {
+ conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L);
+ conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
+ conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
+ loadBalancer.setConf(conf);
+ int numNodes = 30;
+ int numRegions = numNodes * 30;
+ int replication = 3; // 3 replicas per region
+ int numRegionsPerServer = 28;
+ int numTables = 10;
+ int numRacks = 4; // all replicas should be on a different rack
+ Map<ServerName, List<HRegionInfo>> serverMap =
+ createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
+ RackManager rm = new ForTestRackManager(numRacks);
+
+ testWithCluster(serverMap, rm, false, true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2fba25b6/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
new file mode 100644
index 0000000..976ba45
--- /dev/null
+++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java
@@ -0,0 +1,39 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.jruby.embed.PathType;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ClientTests.class, LargeTests.class })
+public class TestShell extends AbstractTestShell {
+
+ @Test
+ public void testRunShellTests() throws IOException {
+ System.setProperty("shell.test.exclude", "replication_admin_test.rb");
+ // Start all ruby tests
+ jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
+ }
+
+}
[08/29] hbase git commit: HBASE-14030 Revert due to pending review
comments
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 2ceeda5..9d9cee0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -85,9 +85,6 @@ public class WALPlayer extends Configured implements Tool {
private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
- public WALPlayer(){
- }
-
protected WALPlayer(final Configuration c) {
super(c);
}
@@ -97,7 +94,7 @@ public class WALPlayer extends Configured implements Tool {
* This one can be used together with {@link KeyValueSortReducer}
*/
static class WALKeyValueMapper
- extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> {
+ extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> {
private byte[] table;
@Override
@@ -109,9 +106,7 @@ public class WALPlayer extends Configured implements Tool {
if (Bytes.equals(table, key.getTablename().getName())) {
for (Cell cell : value.getCells()) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
- if (WALEdit.isMetaEditFamily(kv)) {
- continue;
- }
+ if (WALEdit.isMetaEditFamily(kv)) continue;
context.write(new ImmutableBytesWritable(CellUtil.cloneRow(kv)), kv);
}
}
@@ -137,7 +132,7 @@ public class WALPlayer extends Configured implements Tool {
* a running HBase instance.
*/
protected static class WALMapper
- extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation> {
+ extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation> {
private Map<TableName, TableName> tables = new TreeMap<TableName, TableName>();
@Override
@@ -154,9 +149,7 @@ public class WALPlayer extends Configured implements Tool {
Cell lastCell = null;
for (Cell cell : value.getCells()) {
// filtering WAL meta entries
- if (WALEdit.isMetaEditFamily(cell)) {
- continue;
- }
+ if (WALEdit.isMetaEditFamily(cell)) continue;
// Allow a subclass filter out this cell.
if (filter(context, cell)) {
@@ -167,12 +160,8 @@ public class WALPlayer extends Configured implements Tool {
if (lastCell == null || lastCell.getTypeByte() != cell.getTypeByte()
|| !CellUtil.matchingRow(lastCell, cell)) {
// row or type changed, write out aggregate KVs.
- if (put != null) {
- context.write(tableOut, put);
- }
- if (del != null) {
- context.write(tableOut, del);
- }
+ if (put != null) context.write(tableOut, put);
+ if (del != null) context.write(tableOut, del);
if (CellUtil.isDelete(cell)) {
del = new Delete(CellUtil.cloneRow(cell));
} else {
@@ -188,12 +177,8 @@ public class WALPlayer extends Configured implements Tool {
lastCell = cell;
}
// write residual KVs
- if (put != null) {
- context.write(tableOut, put);
- }
- if (del != null) {
- context.write(tableOut, del);
- }
+ if (put != null) context.write(tableOut, put);
+ if (del != null) context.write(tableOut, del);
}
} catch (InterruptedException e) {
e.printStackTrace();
@@ -201,8 +186,7 @@ public class WALPlayer extends Configured implements Tool {
}
/**
- * Filter cell
- * @param cell cell
+ * @param cell
* @return Return true if we are to emit this cell.
*/
protected boolean filter(Context context, final Cell cell) {
@@ -213,7 +197,9 @@ public class WALPlayer extends Configured implements Tool {
public void setup(Context context) throws IOException {
String[] tableMap = context.getConfiguration().getStrings(TABLE_MAP_KEY);
String[] tablesToUse = context.getConfiguration().getStrings(TABLES_KEY);
- if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) {
+ if (tablesToUse == null && tableMap == null) {
+ // Then user wants all tables.
+ } else if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) {
// this can only happen when WALMapper is used directly by a class other than WALPlayer
throw new IOException("No tables or incorrect table mapping specified.");
}
@@ -229,9 +215,7 @@ public class WALPlayer extends Configured implements Tool {
void setupTime(Configuration conf, String option) throws IOException {
String val = conf.get(option);
- if (null == val) {
- return;
- }
+ if (null == val) return;
long ms;
try {
// first try to parse in user friendly form
@@ -311,8 +295,7 @@ public class WALPlayer extends Configured implements Tool {
return job;
}
- /**
- * Print usage
+ /*
* @param errorMsg Error message. Can be null.
*/
private void usage(final String errorMsg) {
@@ -322,8 +305,7 @@ public class WALPlayer extends Configured implements Tool {
System.err.println("Usage: " + NAME + " [options] <wal inputdir> <tables> [<tableMappings>]");
System.err.println("Read all WAL entries for <tables>.");
System.err.println("If no tables (\"\") are specific, all tables are imported.");
- System.err.println("(Careful, even -ROOT- and hbase:meta entries will be imported"+
- " in that case.)");
+ System.err.println("(Careful, even -ROOT- and hbase:meta entries will be imported in that case.)");
System.err.println("Otherwise <tables> is a comma separated list of tables.\n");
System.err.println("The WAL entries can be mapped to new set of tables via <tableMapping>.");
System.err.println("<tableMapping> is a command separated list of targettables.");
@@ -336,10 +318,10 @@ public class WALPlayer extends Configured implements Tool {
System.err.println(" -D" + WALInputFormat.START_TIME_KEY + "=[date|ms]");
System.err.println(" -D" + WALInputFormat.END_TIME_KEY + "=[date|ms]");
System.err.println(" -D " + JOB_NAME_CONF_KEY
- + "=jobName - use the specified mapreduce job name for the wal player");
+ + "=jobName - use the specified mapreduce job name for the wal player");
System.err.println("For performance also consider the following options:\n"
- + " -Dmapreduce.map.speculative=false\n"
- + " -Dmapreduce.reduce.speculative=false");
+ + " -Dmapreduce.map.speculative=false\n"
+ + " -Dmapreduce.reduce.speculative=false");
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 5cd38b5..bdb19f4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -75,7 +75,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
-import org.apache.hadoop.hbase.backup.BackupManager;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
@@ -385,7 +384,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
Replication.decorateMasterConfiguration(this.conf);
- BackupManager.decorateMasterConfiguration(this.conf);
// Hack! Maps DFSClient => Master for logs. HDFS made this
// config param for task trackers, but we can piggyback off of it.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
index b6e11ea..95c3ffe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java
@@ -37,7 +37,7 @@ public abstract class RegionServerProcedureManager extends ProcedureManager {
* @param rss Region Server service interface
* @throws KeeperException
*/
- public abstract void initialize(RegionServerServices rss) throws IOException;
+ public abstract void initialize(RegionServerServices rss) throws KeeperException;
/**
* Start accepting procedure requests.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
index adb3604..0f4ea64 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.procedure.flush.RegionServerFlushTableProcedureManager;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
+import org.apache.zookeeper.KeeperException;
/**
* Provides the globally barriered procedure framework and environment
@@ -38,7 +39,7 @@ public class RegionServerProcedureManagerHost extends
private static final Log LOG = LogFactory
.getLog(RegionServerProcedureManagerHost.class);
- public void initialize(RegionServerServices rss) throws IOException {
+ public void initialize(RegionServerServices rss) throws KeeperException {
for (RegionServerProcedureManager proc : procedures) {
LOG.debug("Procedure " + proc.getProcedureSignature() + " is initializing");
proc.initialize(rss);
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
index 3865ba9..085d642 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java
@@ -54,7 +54,7 @@ public class ZKProcedureCoordinatorRpcs implements ProcedureCoordinatorRpcs {
* @throws KeeperException if an unexpected zk error occurs
*/
public ZKProcedureCoordinatorRpcs(ZooKeeperWatcher watcher,
- String procedureClass, String coordName) throws IOException {
+ String procedureClass, String coordName) throws KeeperException {
this.watcher = watcher;
this.procedureType = procedureClass;
this.coordName = coordName;
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index fff75a7..2e03a60 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -68,54 +68,49 @@ public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
* @throws KeeperException if we can't reach zookeeper
*/
public ZKProcedureMemberRpcs(final ZooKeeperWatcher watcher, final String procType)
- throws IOException {
- try {
- this.zkController = new ZKProcedureUtil(watcher, procType) {
- @Override
- public void nodeCreated(String path) {
- if (!isInProcedurePath(path)) {
- return;
- }
+ throws KeeperException {
+ this.zkController = new ZKProcedureUtil(watcher, procType) {
+ @Override
+ public void nodeCreated(String path) {
+ if (!isInProcedurePath(path)) {
+ return;
+ }
- LOG.info("Received created event:" + path);
- // if it is a simple start/end/abort then we just rewatch the node
- if (isAcquiredNode(path)) {
- waitForNewProcedures();
- return;
- } else if (isAbortNode(path)) {
- watchForAbortedProcedures();
- return;
- }
- String parent = ZKUtil.getParent(path);
- // if its the end barrier, the procedure can be completed
- if (isReachedNode(parent)) {
- receivedReachedGlobalBarrier(path);
- return;
- } else if (isAbortNode(parent)) {
- abort(path);
- return;
- } else if (isAcquiredNode(parent)) {
- startNewSubprocedure(path);
- } else {
- LOG.debug("Ignoring created notification for node:" + path);
- }
+ LOG.info("Received created event:" + path);
+ // if it is a simple start/end/abort then we just rewatch the node
+ if (isAcquiredNode(path)) {
+ waitForNewProcedures();
+ return;
+ } else if (isAbortNode(path)) {
+ watchForAbortedProcedures();
+ return;
}
+ String parent = ZKUtil.getParent(path);
+ // if its the end barrier, the procedure can be completed
+ if (isReachedNode(parent)) {
+ receivedReachedGlobalBarrier(path);
+ return;
+ } else if (isAbortNode(parent)) {
+ abort(path);
+ return;
+ } else if (isAcquiredNode(parent)) {
+ startNewSubprocedure(path);
+ } else {
+ LOG.debug("Ignoring created notification for node:" + path);
+ }
+ }
- @Override
- public void nodeChildrenChanged(String path) {
- if (path.equals(this.acquiredZnode)) {
- LOG.info("Received procedure start children changed event: " + path);
- waitForNewProcedures();
- } else if (path.equals(this.abortZnode)) {
- LOG.info("Received procedure abort children changed event: " + path);
- watchForAbortedProcedures();
- }
+ @Override
+ public void nodeChildrenChanged(String path) {
+ if (path.equals(this.acquiredZnode)) {
+ LOG.info("Received procedure start children changed event: " + path);
+ waitForNewProcedures();
+ } else if (path.equals(this.abortZnode)) {
+ LOG.info("Received procedure abort children changed event: " + path);
+ watchForAbortedProcedures();
}
- };
- } catch (KeeperException e) {
- // TODO Auto-generated catch block
- throw new IOException(e);
- }
+ }
+ };
}
public ZKProcedureUtil getZkController() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
index bd65cc7..1aa959c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java
@@ -317,7 +317,7 @@ public class RegionServerFlushTableProcedureManager extends RegionServerProcedur
* @throws KeeperException if the zookeeper cannot be reached
*/
@Override
- public void initialize(RegionServerServices rss) throws IOException {
+ public void initialize(RegionServerServices rss) throws KeeperException {
this.rss = rss;
ZooKeeperWatcher zkw = rss.getZooKeeper();
this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1cd54fa..211fed5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -807,8 +807,8 @@ public class HRegionServer extends HasThread implements
rspmHost = new RegionServerProcedureManagerHost();
rspmHost.loadProcedures(conf);
rspmHost.initialize(this);
- } catch (IOException e) {
- this.abort("Failed to reach coordination cluster when creating procedure handler.", e);
+ } catch (KeeperException e) {
+ this.abort("Failed to reach zk cluster when creating procedure handler.", e);
}
// register watcher for recovering regions
this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this);
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
index e56dd28..537329a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java
@@ -390,7 +390,7 @@ public class RegionServerSnapshotManager extends RegionServerProcedureManager {
* @throws KeeperException if the zookeeper cluster cannot be reached
*/
@Override
- public void initialize(RegionServerServices rss) throws IOException {
+ public void initialize(RegionServerServices rss) throws KeeperException {
this.rss = rss;
ZooKeeperWatcher zkw = rss.getZooKeeper();
this.memberRpcs = new ZKProcedureMemberRpcs(zkw,
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index acde21e..9ae72e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -97,8 +97,6 @@ import com.lmax.disruptor.TimeoutException;
import com.lmax.disruptor.dsl.Disruptor;
import com.lmax.disruptor.dsl.ProducerType;
-
-
/**
* Implementation of {@link WAL} to go against {@link FileSystem}; i.e. keep WALs in HDFS.
* Only one WAL is ever being written at a time. When a WAL hits a configured maximum size,
@@ -361,9 +359,7 @@ public class FSHLog implements WAL {
public int compare(Path o1, Path o2) {
long t1 = getFileNumFromFileName(o1);
long t2 = getFileNumFromFileName(o2);
- if (t1 == t2) {
- return 0;
- }
+ if (t1 == t2) return 0;
return (t1 > t2) ? 1 : -1;
}
};
@@ -406,7 +402,7 @@ public class FSHLog implements WAL {
* @param root path for stored and archived wals
* @param logDir dir where wals are stored
* @param conf configuration to use
- * @throws IOException exception
+ * @throws IOException
*/
public FSHLog(final FileSystem fs, final Path root, final String logDir, final Configuration conf)
throws IOException {
@@ -414,7 +410,7 @@ public class FSHLog implements WAL {
}
/**
- * Create an edit log at the given directory location.
+ * Create an edit log at the given <code>dir</code> location.
*
* You should never have to load an existing log. If there is a log at
* startup, it should have already been processed and deleted by the time the
@@ -429,13 +425,13 @@ public class FSHLog implements WAL {
* be registered before we do anything else; e.g. the
* Constructor {@link #rollWriter()}.
* @param failIfWALExists If true IOException will be thrown if files related to this wal
- * already exist.
+ * already exist.
* @param prefix should always be hostname and port in distributed env and
- * it will be URL encoded before being used.
- * If prefix is null, "wal" will be used
+ * it will be URL encoded before being used.
+ * If prefix is null, "wal" will be used
* @param suffix will be url encoded. null is treated as empty. non-empty must start with
- * {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER}
- * @throws IOException exception
+ * {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER}
+ * @throws IOException
*/
public FSHLog(final FileSystem fs, final Path rootDir, final String logDir,
final String archiveDir, final Configuration conf,
@@ -597,9 +593,7 @@ public class FSHLog implements WAL {
@VisibleForTesting
OutputStream getOutputStream() {
FSDataOutputStream fsdos = this.hdfs_out;
- if (fsdos == null) {
- return null;
- }
+ if (fsdos == null) return null;
return fsdos.getWrappedStream();
}
@@ -634,7 +628,7 @@ public class FSHLog implements WAL {
/**
* Tell listeners about pre log roll.
- * @throws IOException exception
+ * @throws IOException
*/
private void tellListenersAboutPreLogRoll(final Path oldPath, final Path newPath)
throws IOException {
@@ -647,7 +641,7 @@ public class FSHLog implements WAL {
/**
* Tell listeners about post log roll.
- * @throws IOException exception
+ * @throws IOException
*/
private void tellListenersAboutPostLogRoll(final Path oldPath, final Path newPath)
throws IOException {
@@ -660,7 +654,8 @@ public class FSHLog implements WAL {
/**
* Run a sync after opening to set up the pipeline.
- * @param nextWriter next writer
+ * @param nextWriter
+ * @param startTimeNanos
*/
private void preemptiveSync(final ProtobufLogWriter nextWriter) {
long startTimeNanos = System.nanoTime();
@@ -678,9 +673,7 @@ public class FSHLog implements WAL {
rollWriterLock.lock();
try {
// Return if nothing to flush.
- if (!force && (this.writer != null && this.numEntries.get() <= 0)) {
- return null;
- }
+ if (!force && (this.writer != null && this.numEntries.get() <= 0)) return null;
byte [][] regionsToFlush = null;
if (this.closed) {
LOG.debug("WAL closed. Skipping rolling of writer");
@@ -735,7 +728,7 @@ public class FSHLog implements WAL {
/**
* Archive old logs. A WAL is eligible for archiving if all its WALEdits have been flushed.
- * @throws IOException exception
+ * @throws IOException
*/
private void cleanOldLogs() throws IOException {
List<Path> logsToArchive = null;
@@ -745,13 +738,9 @@ public class FSHLog implements WAL {
Path log = e.getKey();
Map<byte[], Long> sequenceNums = e.getValue();
if (this.sequenceIdAccounting.areAllLower(sequenceNums)) {
- if (logsToArchive == null) {
- logsToArchive = new ArrayList<Path>();
- }
+ if (logsToArchive == null) logsToArchive = new ArrayList<Path>();
logsToArchive.add(log);
- if (LOG.isTraceEnabled()) {
- LOG.trace("WAL file ready for archiving " + log);
- }
+ if (LOG.isTraceEnabled()) LOG.trace("WAL file ready for archiving " + log);
}
}
if (logsToArchive != null) {
@@ -781,9 +770,7 @@ public class FSHLog implements WAL {
if (regions != null) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < regions.length; i++) {
- if (i > 0) {
- sb.append(", ");
- }
+ if (i > 0) sb.append(", ");
sb.append(Bytes.toStringBinary(regions[i]));
}
LOG.info("Too many WALs; count=" + logCount + ", max=" + this.maxLogs +
@@ -849,9 +836,7 @@ public class FSHLog implements WAL {
}
} catch (FailedSyncBeforeLogCloseException e) {
// If unflushed/unsynced entries on close, it is reason to abort.
- if (isUnflushedEntries()) {
- throw e;
- }
+ if (isUnflushedEntries()) throw e;
LOG.warn("Failed sync-before-close but no outstanding appends; closing WAL: " +
e.getMessage());
}
@@ -912,9 +897,7 @@ public class FSHLog implements WAL {
try {
blockOnSync(syncFuture);
} catch (IOException ioe) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Stale sync exception", ioe);
- }
+ if (LOG.isTraceEnabled()) LOG.trace("Stale sync exception", ioe);
}
}
}
@@ -985,15 +968,7 @@ public class FSHLog implements WAL {
public Path getCurrentFileName() {
return computeFilename(this.filenum.get());
}
-
- /**
- * To support old API compatibility
- * @return current file number (timestamp)
- */
- public long getFilenum() {
- return filenum.get();
- }
-
+
@Override
public String toString() {
return "FSHLog " + logFilePrefix + ":" + logFileSuffix + "(num " + filenum + ")";
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java
deleted file mode 100644
index 0360000..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import org.apache.hadoop.hbase.backup.BackupHandler;
-
-/* this class will be extended in future jira to support progress report */
-public class SnapshotCopy extends ExportSnapshot {
- private BackupHandler backupHandler;
- private String table;
-
- public SnapshotCopy(BackupHandler backupHandler, String table) {
- super();
- this.backupHandler = backupHandler;
- this.table = table;
- }
-
- public BackupHandler getBackupHandler() {
- return this.backupHandler;
- }
-
- public String getTable() {
- return this.table;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
index dd4d337..027e7a2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
@@ -209,18 +209,13 @@ public class DefaultWALProvider implements WALProvider {
@VisibleForTesting
public static long extractFileNumFromWAL(final WAL wal) {
final Path walName = ((FSHLog)wal).getCurrentFileName();
- return extractFileNumFromWAL(walName);
- }
-
- @VisibleForTesting
- public static long extractFileNumFromWAL(final Path walName) {
if (walName == null) {
throw new IllegalArgumentException("The WAL path couldn't be null");
}
final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
}
-
+
/**
* Pattern used to validate a WAL file name
* see {@link #validateWALFilename(String)} for description.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
deleted file mode 100644
index bc0c848..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-/**
- * This class is only a base for other integration-level backup tests.
- * Do not add tests here.
- * TestBackupSmallTests is where tests that don't require bring machines up/down should go
- * All other tests should have their own classes and extend this one
- */
-public class TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestBackupBase.class);
-
- protected static Configuration conf1;
- protected static Configuration conf2;
-
- protected static HBaseTestingUtility TEST_UTIL;
- protected static HBaseTestingUtility TEST_UTIL2;
-
- protected static TableName table1;
- protected static TableName table2;
- protected static TableName table3;
- protected static TableName table4;
-
- protected static String table1_restore = "table1_restore";
- protected static String table2_restore = "table2_restore";
- protected static String table3_restore = "table3_restore";
- protected static String table4_restore = "table4_restore";
-
- protected static final int NB_ROWS_IN_BATCH = 100;
- protected static final byte[] qualName = Bytes.toBytes("q1");
- protected static final byte[] famName = Bytes.toBytes("f");
-
- protected static String BACKUP_ROOT_DIR = "/backupUT";
- protected static String BACKUP_REMOTE_ROOT_DIR = "/backupUT";
-
- protected static final String BACKUP_ZNODE = "/backup/hbase";
- protected static final String BACKUP_SUCCEED_NODE = "complete";
- protected static final String BACKUP_FAILED_NODE = "failed";
-
-
- /**
- * @throws java.lang.Exception
- */
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
-
- TEST_UTIL = new HBaseTestingUtility();
- TEST_UTIL.getConfiguration().set("hbase.procedure.regionserver.classes",
- LogRollRegionServerProcedureManager.class.getName());
- TEST_UTIL.getConfiguration().set("hbase.procedure.master.classes",
- LogRollMasterProcedureManager.class.getName());
- TEST_UTIL.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
- TEST_UTIL.startMiniZKCluster();
- MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
-
- conf1 = TEST_UTIL.getConfiguration();
- conf2 = HBaseConfiguration.create(conf1);
- conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
- TEST_UTIL2 = new HBaseTestingUtility(conf2);
- TEST_UTIL2.setZkCluster(miniZK);
- TEST_UTIL.startMiniCluster();
- TEST_UTIL2.startMiniCluster();
- conf1 = TEST_UTIL.getConfiguration();
-
- TEST_UTIL.startMiniMapReduceCluster();
- BACKUP_ROOT_DIR = TEST_UTIL.getConfiguration().get("fs.defaultFS") + "/backupUT";
- LOG.info("ROOTDIR " + BACKUP_ROOT_DIR);
- BACKUP_REMOTE_ROOT_DIR = TEST_UTIL2.getConfiguration().get("fs.defaultFS") + "/backupUT";
- LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR);
-
- BackupClient.setConf(conf1);
- RestoreClient.setConf(conf1);
- createTables();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
- SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
- //zkw1.close();
- TEST_UTIL2.shutdownMiniCluster();
- TEST_UTIL.shutdownMiniCluster();
- TEST_UTIL.shutdownMiniMapReduceCluster();
- }
-
- protected static void loadTable(HTable table) throws Exception {
-
- Put p; // 100 + 1 row to t1_syncup
- for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
- p = new Put(Bytes.toBytes("row" + i));
- p.addColumn(famName, qualName, Bytes.toBytes("val" + i));
- table.put(p);
- }
- }
-
- protected static void createTables() throws Exception {
-
- long tid = System.currentTimeMillis();
- table1 = TableName.valueOf("test-" + tid);
- HBaseAdmin ha = TEST_UTIL.getHBaseAdmin();
- HTableDescriptor desc = new HTableDescriptor(table1);
- HColumnDescriptor fam = new HColumnDescriptor(famName);
- desc.addFamily(fam);
- ha.createTable(desc);
- Connection conn = ConnectionFactory.createConnection(conf1);
- HTable table = (HTable) conn.getTable(table1);
- loadTable(table);
- table.close();
- table2 = TableName.valueOf("test-" + tid + 1);
- desc = new HTableDescriptor(table2);
- desc.addFamily(fam);
- ha.createTable(desc);
- table = (HTable) conn.getTable(table2);
- loadTable(table);
- table.close();
- table3 = TableName.valueOf("test-" + tid + 2);
- table = TEST_UTIL.createTable(table3, famName);
- table.close();
- table4 = TableName.valueOf("test-" + tid + 3);
- table = TEST_UTIL.createTable(table4, famName);
- table.close();
- ha.close();
- conn.close();
- }
-
- protected boolean checkSucceeded(String backupId) throws IOException
- {
- BackupContext status = getBackupContext(backupId);
- if(status == null) return false;
- return status.getFlag() == BACKUPSTATUS.COMPLETE;
- }
-
- protected boolean checkFailed(String backupId) throws IOException
- {
- BackupContext status = getBackupContext(backupId);
- if(status == null) return false;
- return status.getFlag() == BACKUPSTATUS.FAILED;
- }
-
- private BackupContext getBackupContext(String backupId) throws IOException
- {
- Configuration conf = BackupClient.getConf();
- BackupSystemTable table = BackupSystemTable.getTable(conf);
- BackupContext status = table.readBackupStatus(backupId);
- return status;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java
deleted file mode 100644
index 8be07bc..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestBackupBoundaryTests extends TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestBackupBoundaryTests.class);
-
- /**
- * Verify that full backup is created on a single empty table correctly.
- * @throws Exception
- */
- @Test
- public void testFullBackupSingleEmpty() throws Exception {
-
- LOG.info("create full backup image on single table");
-
- String backupId =
- BackupClient.create("full", BACKUP_ROOT_DIR, table3.getNameAsString(), null);
- LOG.info("Finished Backup");
- assertTrue(checkSucceeded(backupId));
- }
-
- /**
- * Verify that full backup is created on multiple empty tables correctly.
- * @throws Exception
- */
- @Test
- public void testFullBackupMultipleEmpty() throws Exception {
- LOG.info("create full backup image on mulitple empty tables");
- String tableset =
- table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table4.getNameAsString();
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- assertTrue(checkSucceeded(backupId));
-
- }
-
- /**
- * Verify that full backup fails on a single table that does not exist.
- * @throws Exception
- */
- @Test(expected = RuntimeException.class)
- public void testFullBackupSingleDNE() throws Exception {
-
- LOG.info("test full backup fails on a single table that does not exist");
- BackupClient.create("full", BACKUP_ROOT_DIR, "tabledne", null);
- }
-
- /**
- * Verify that full backup fails on multiple tables that do not exist.
- * @throws Exception
- */
- @Test(expected = RuntimeException.class)
- public void testFullBackupMultipleDNE() throws Exception {
-
- LOG.info("test full backup fails on multiple tables that do not exist");
- BackupClient.create("full", BACKUP_ROOT_DIR, "table1dne,table2dne", null);
- }
-
- /**
- * Verify that full backup fails on tableset containing real and fake tables.
- * @throws Exception
- */
- @Test(expected = RuntimeException.class)
- public void testFullBackupMixExistAndDNE() throws Exception {
- LOG.info("create full backup fails on tableset containing real and fake table");
- String tableset =
- table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + "tabledne";
- BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java
deleted file mode 100644
index 158479b..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.DefaultWALProvider;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import com.google.common.collect.Iterables;
-
-@Category(LargeTests.class)
-public class TestBackupLogCleaner extends TestBackupBase {
- private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class);
-
- // implements all test cases in 1 test since incremental full backup/
- // incremental backup has dependencies
- @Test
- public void testBackupLogCleaner() throws Exception {
-
- // #1 - create full backup for all tables
- LOG.info("create full backup image for all tables");
- String tablesetFull =
- table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table4.getNameAsString();
-
- BackupSystemTable systemTable = BackupSystemTable.getTable(TEST_UTIL.getConfiguration());
- // Verify that we have no backup sessions yet
- assertFalse(systemTable.hasBackupSessions());
-
- List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
- List<String> swalFiles = convert(walFiles);
- BackupLogCleaner cleaner = new BackupLogCleaner();
- cleaner.setConf(TEST_UTIL.getConfiguration());
-
- Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles);
- // We can delete all files because we do not have yet recorded backup sessions
- assertTrue(Iterables.size(deletable) == walFiles.size());
-
- systemTable.addWALFiles(swalFiles, "backup");
- String backupIdFull = BackupClient.create("full", BACKUP_ROOT_DIR, tablesetFull, null);
- assertTrue(checkSucceeded(backupIdFull));
- // Check one more time
- deletable = cleaner.getDeletableFiles(walFiles);
- // We can delete wal files because they were saved into hbase:backup table
- int size = Iterables.size(deletable);
- assertTrue(size == walFiles.size());
-
- List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
- LOG.debug("WAL list after full backup");
- convert(newWalFiles);
-
- // New list of wal files is greater than the previous one,
- // because new wal per RS have been opened after full backup
- assertTrue(walFiles.size() < newWalFiles.size());
- // TODO : verify that result files are not walFiles collection
- Connection conn = ConnectionFactory.createConnection(conf1);
- // #2 - insert some data to table
- HTable t1 = (HTable) conn.getTable(table1);
- Put p1;
- for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
- p1 = new Put(Bytes.toBytes("row-t1" + i));
- p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
- t1.put(p1);
- }
-
- t1.close();
-
- HTable t2 = (HTable) conn.getTable(table2);
- Put p2;
- for (int i = 0; i < 5; i++) {
- p2 = new Put(Bytes.toBytes("row-t2" + i));
- p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
- t2.put(p2);
- }
-
- t2.close();
-
- // #3 - incremental backup for multiple tables
- String tablesetIncMultiple =
- table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString();
-
- String backupIdIncMultiple =
- BackupClient.create("incremental", BACKUP_ROOT_DIR, tablesetIncMultiple, null);
- assertTrue(checkSucceeded(backupIdIncMultiple));
- deletable = cleaner.getDeletableFiles(newWalFiles);
-
- assertTrue(Iterables.size(deletable) == newWalFiles.size());
-
- conn.close();
-
- }
-
- private List<String> convert(List<FileStatus> walFiles) {
- List<String> result = new ArrayList<String>();
- for (FileStatus fs : walFiles) {
- LOG.debug("+++WAL: " + fs.getPath().toString());
- result.add(fs.getPath().toString());
- }
- return result;
- }
-
- private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
- Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME);
- FileSystem fs = FileSystem.get(c);
- RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
- List<FileStatus> logFiles = new ArrayList<FileStatus>();
- while (it.hasNext()) {
- LocatedFileStatus lfs = it.next();
- if (lfs.isFile() && !DefaultWALProvider.isMetaFile(lfs.getPath())) {
- logFiles.add(lfs);
- LOG.info(lfs);
- }
- }
- return logFiles;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java
deleted file mode 100644
index 7421707..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
-import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test cases for hbase:backup API
- *
- */
-@Category(MediumTests.class)
-public class TestBackupSystemTable {
-
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
- protected static Configuration conf = UTIL.getConfiguration();
- protected static MiniHBaseCluster cluster;
-
- @BeforeClass
- public static void setUp() throws Exception {
- cluster = UTIL.startMiniCluster();
-
- }
-
- @Test
- public void testUpdateReadDeleteBackupStatus() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
- BackupContext ctx = createBackupContext();
- table.updateBackupStatus(ctx);
- BackupContext readCtx = table.readBackupStatus(ctx.getBackupId());
- assertTrue(compare(ctx, readCtx));
-
- // try fake backup id
- readCtx = table.readBackupStatus("fake");
-
- assertNull(readCtx);
- // delete backup context
- table.deleteBackupStatus(ctx.getBackupId());
- readCtx = table.readBackupStatus(ctx.getBackupId());
- assertNull(readCtx);
- cleanBackupTable();
- }
-
- @Test
- public void testWriteReadBackupStartCode() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
- String code = "100";
- table.writeBackupStartCode(code);
- String readCode = table.readBackupStartCode();
- assertEquals(code, readCode);
- cleanBackupTable();
- }
-
- private void cleanBackupTable() throws IOException {
- Admin admin = UTIL.getHBaseAdmin();
- admin.disableTable(BackupSystemTable.getTableName());
- admin.truncateTable(BackupSystemTable.getTableName(), true);
- if (admin.isTableDisabled(BackupSystemTable.getTableName())) {
- admin.enableTable(BackupSystemTable.getTableName());
- }
- }
-
- @Test
- public void testBackupHistory() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
- int n = 10;
- List<BackupContext> list = createBackupContextList(n);
-
- // Load data
- for (BackupContext bc : list) {
- // Make sure we set right status
- bc.setFlag(BACKUPSTATUS.COMPLETE);
- table.updateBackupStatus(bc);
- }
-
- // Reverse list for comparison
- Collections.reverse(list);
- ArrayList<BackupCompleteData> history = table.getBackupHistory();
- assertTrue(history.size() == n);
-
- for (int i = 0; i < n; i++) {
- BackupContext ctx = list.get(i);
- BackupCompleteData data = history.get(i);
- assertTrue(compare(ctx, data));
- }
-
- cleanBackupTable();
-
- }
-
- @Test
- public void testRegionServerLastLogRollResults() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
-
- String[] servers = new String[] { "server1", "server2", "server3" };
- String[] timestamps = new String[] { "100", "102", "107" };
-
- for (int i = 0; i < servers.length; i++) {
- table.writeRegionServerLastLogRollResult(servers[i], timestamps[i]);
- }
-
- HashMap<String, String> result = table.readRegionServerLastLogRollResult();
- assertTrue(servers.length == result.size());
- Set<String> keys = result.keySet();
- String[] keysAsArray = new String[keys.size()];
- keys.toArray(keysAsArray);
- Arrays.sort(keysAsArray);
-
- for (int i = 0; i < keysAsArray.length; i++) {
- assertEquals(keysAsArray[i], servers[i]);
- String ts1 = timestamps[i];
- String ts2 = result.get(keysAsArray[i]);
- assertEquals(ts1, ts2);
- }
-
- cleanBackupTable();
-
- }
-
- @Test
- public void testIncrementalBackupTableSet() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
-
- TreeSet<String> tables1 = new TreeSet<String>();
-
- tables1.add("t1");
- tables1.add("t2");
- tables1.add("t3");
-
- TreeSet<String> tables2 = new TreeSet<String>();
-
- tables2.add("t3");
- tables2.add("t4");
- tables2.add("t5");
-
- table.addIncrementalBackupTableSet(tables1);
- TreeSet<String> res1 = (TreeSet<String>) table.getIncrementalBackupTableSet();
- assertTrue(tables1.size() == res1.size());
- Iterator<String> desc1 = tables1.descendingIterator();
- Iterator<String> desc2 = res1.descendingIterator();
- while (desc1.hasNext()) {
- assertEquals(desc1.next(), desc2.next());
- }
-
- table.addIncrementalBackupTableSet(tables2);
- TreeSet<String> res2 = (TreeSet<String>) table.getIncrementalBackupTableSet();
- assertTrue((tables2.size() + tables1.size() - 1) == res2.size());
-
- tables1.addAll(tables2);
-
- desc1 = tables1.descendingIterator();
- desc2 = res2.descendingIterator();
-
- while (desc1.hasNext()) {
- assertEquals(desc1.next(), desc2.next());
- }
- cleanBackupTable();
-
- }
-
- @Test
- public void testRegionServerLogTimestampMap() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
-
- TreeSet<String> tables = new TreeSet<String>();
-
- tables.add("t1");
- tables.add("t2");
- tables.add("t3");
-
- HashMap<String, String> rsTimestampMap = new HashMap<String, String>();
-
- rsTimestampMap.put("rs1", "100");
- rsTimestampMap.put("rs2", "101");
- rsTimestampMap.put("rs3", "103");
-
- table.writeRegionServerLogTimestamp(tables, rsTimestampMap);
-
- HashMap<String, HashMap<String, String>> result = table.readLogTimestampMap();
-
- assertTrue(tables.size() == result.size());
-
- for (String t : tables) {
- HashMap<String, String> rstm = result.get(t);
- assertNotNull(rstm);
- assertEquals(rstm.get("rs1"), "100");
- assertEquals(rstm.get("rs2"), "101");
- assertEquals(rstm.get("rs3"), "103");
- }
-
- Set<String> tables1 = new TreeSet<String>();
-
- tables1.add("t3");
- tables1.add("t4");
- tables1.add("t5");
-
- HashMap<String, String> rsTimestampMap1 = new HashMap<String, String>();
-
- rsTimestampMap1.put("rs1", "200");
- rsTimestampMap1.put("rs2", "201");
- rsTimestampMap1.put("rs3", "203");
-
- table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1);
-
- result = table.readLogTimestampMap();
-
- assertTrue(5 == result.size());
-
- for (String t : tables) {
- HashMap<String, String> rstm = result.get(t);
- assertNotNull(rstm);
- if (t.equals("t3") == false) {
- assertEquals(rstm.get("rs1"), "100");
- assertEquals(rstm.get("rs2"), "101");
- assertEquals(rstm.get("rs3"), "103");
- } else {
- assertEquals(rstm.get("rs1"), "200");
- assertEquals(rstm.get("rs2"), "201");
- assertEquals(rstm.get("rs3"), "203");
- }
- }
-
- for (String t : tables1) {
- HashMap<String, String> rstm = result.get(t);
- assertNotNull(rstm);
- assertEquals(rstm.get("rs1"), "200");
- assertEquals(rstm.get("rs2"), "201");
- assertEquals(rstm.get("rs3"), "203");
- }
-
- cleanBackupTable();
-
- }
-
- @Test
- public void testAddWALFiles() throws IOException {
- BackupSystemTable table = BackupSystemTable.getTable(conf);
- FileSystem fs = FileSystem.get(conf);
- List<String> files =
- Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1",
- "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2",
- "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3");
- String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5";
-
- table.addWALFiles(files, "backup");
-
- assertTrue(table.checkWALFile(files.get(0)));
- assertTrue(table.checkWALFile(files.get(1)));
- assertTrue(table.checkWALFile(files.get(2)));
- assertFalse(table.checkWALFile(newFile));
-
- cleanBackupTable();
- }
-
- private boolean compare(BackupContext ctx, BackupCompleteData data) {
-
- return ctx.getBackupId().equals(data.getBackupToken())
- && ctx.getTargetRootDir().equals(data.getBackupRootPath())
- && ctx.getType().equals(data.getType())
- && ctx.getStartTs() == Long.parseLong(data.getStartTime())
- && ctx.getEndTs() == Long.parseLong(data.getEndTime());
-
- }
-
- private boolean compare(BackupContext one, BackupContext two) {
- return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType())
- && one.getTargetRootDir().equals(two.getTargetRootDir())
- && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs();
- }
-
- private BackupContext createBackupContext() {
-
- BackupContext ctxt =
- new BackupContext("backup_" + System.nanoTime(), "full", new String[] { "t1", "t2", "t3" },
- "/hbase/backup", null);
- ctxt.setStartTs(System.currentTimeMillis());
- ctxt.setEndTs(System.currentTimeMillis() + 1);
- return ctxt;
- }
-
- private List<BackupContext> createBackupContextList(int size) {
- List<BackupContext> list = new ArrayList<BackupContext>();
- for (int i = 0; i < size; i++) {
- list.add(createBackupContext());
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- return list;
- }
-
- @AfterClass
- public static void tearDown() throws IOException {
- if (cluster != null) cluster.shutdown();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java
deleted file mode 100644
index 19caf37..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestFullBackup extends TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestFullBackup.class);
-
- /**
- * Verify that full backup is created on a single table with data correctly.
- * @throws Exception
- */
- @Test
- public void testFullBackupSingle() throws Exception {
-
- LOG.info("test full backup on a single table with data");
- String backupId =
- BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
- }
-
- /**
- * Verify that full backup is created on multiple tables correctly.
- * @throws Exception
- */
- @Test
- public void testFullBackupMultiple() throws Exception {
- LOG.info("create full backup image on multiple tables with data");
- String tableset =
- table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table2.getNameAsString();
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- assertTrue(checkSucceeded(backupId));
-
- }
-
- /**
- * Verify that full backup is created on all tables correctly.
- * @throws Exception
- */
- @Test
- public void testFullBackupAll() throws Exception {
- LOG.info("create full backup image on all tables");
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, null, null);
- assertTrue(checkSucceeded(backupId));
-
- }
-
- /**
- * Verify that full backup is created on a table correctly using a snapshot.
- * @throws Exception
- */
- //@Test
- //public void testFullBackupUsingSnapshot() throws Exception {
- // HBaseAdmin hba = new HBaseAdmin(conf1);
- //String snapshot = "snapshot";
- //hba.snapshot(snapshot, table1);
- //LOG.info("create full backup image on a table using snapshot");
- //String backupId =
- // BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(),
- // snapshot);
- // }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
deleted file mode 100644
index 1262c54..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestFullRestore extends TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestFullRestore.class);
-
- /**
- * Verify that a single table is restored to a new table
- * @throws Exception
- */
- @Test
- public void testFullRestoreSingle() throws Exception {
-
- LOG.info("test full restore on a single table empty table");
- String backupId =
- BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
-
- String[] tableset = new String[] { table1.getNameAsString() };
- String[] tablemap = new String[] { table1_restore };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap,
- false);
- HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
- assertTrue(hba.tableExists(TableName.valueOf(table1_restore)));
- TEST_UTIL.deleteTable(TableName.valueOf(table1_restore));
- hba.close();
- }
-
- /**
- * Verify that multiple tables are restored to new tables.
- * @throws Exception
- */
- @Test
- public void testFullRestoreMultiple() throws Exception {
- LOG.info("create full backup image on multiple tables");
- String tableset =
- table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString();
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- assertTrue(checkSucceeded(backupId));
-
- String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() };
- String[] tablemap = new String[] { table2_restore, table3_restore };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false,
- restore_tableset, tablemap, false);
- HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
- assertTrue(hba.tableExists(TableName.valueOf(table2_restore)));
- assertTrue(hba.tableExists(TableName.valueOf(table3_restore)));
- TEST_UTIL.deleteTable(TableName.valueOf(table2_restore));
- TEST_UTIL.deleteTable(TableName.valueOf(table3_restore));
- hba.close();
- }
-
- /**
- * Verify that a single table is restored using overwrite
- * @throws Exception
- */
- @Test
- public void testFullRestoreSingleOverwrite() throws Exception {
-
- LOG.info("test full restore on a single table empty table");
- String backupId =
- BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
-
- String[] tableset = new String[] { table1.getNameAsString() };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, null,
- true);
- }
-
- /**
- * Verify that multiple tables are restored to new tables using overwrite.
- * @throws Exception
- */
- @Test
- public void testFullRestoreMultipleOverwrite() throws Exception {
- LOG.info("create full backup image on multiple tables");
- String tableset =
- table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString();
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- assertTrue(checkSucceeded(backupId));
-
- String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false,
- false, restore_tableset, null, true);
- }
-
- /**
- * Verify that restore fails on a single table that does not exist.
- * @throws Exception
- */
- @Test(expected = IOException.class)
- public void testFullRestoreSingleDNE() throws Exception {
-
- LOG.info("test restore fails on a single table that does not exist");
- String backupId =
- BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
-
- String[] tableset = new String[] { "faketable" };
- String[] tablemap = new String[] { table1_restore };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap,
- false);
- }
-
- /**
- * Verify that restore fails on multiple tables that do not exist.
- * @throws Exception
- */
- @Test(expected = IOException.class)
- public void testFullRestoreMultipleDNE() throws Exception {
-
- LOG.info("test restore fails on multiple tables that do not exist");
- String tableset =
- table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString();
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- assertTrue(checkSucceeded(backupId));
-
- String[] restore_tableset = new String[] { "faketable1", "faketable2" };
- String[] tablemap = new String[] { table2_restore, table3_restore };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false,
- false, restore_tableset, tablemap, false);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
deleted file mode 100644
index 5437b84..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.hamcrest.CoreMatchers;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestIncrementalBackup extends TestBackupBase {
- private static final Log LOG = LogFactory.getLog(TestIncrementalBackup.class);
- //implement all testcases in 1 test since incremental backup/restore has dependencies
- @Test
- public void TestIncBackupRestore() throws Exception {
- HBackupFileSystem hbfs;
-
- // #1 - create full backup for all tables
- LOG.info("create full backup image for all tables");
- String tablesetFull =
- table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table4.getNameAsString();
-
- String backupIdFull =
- BackupClient.create("full", BACKUP_ROOT_DIR, tablesetFull, null);
- assertTrue(checkSucceeded(backupIdFull));
-
- Connection conn = ConnectionFactory.createConnection(conf1);
- // #2 - insert some data to table
- HTable t1 = (HTable) conn.getTable(table1);
- Put p1;
- for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
- p1 = new Put(Bytes.toBytes("row-t1" + i));
- p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
- t1.put(p1);
- }
-
- Assert.assertThat(TEST_UTIL.countRows(t1), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2));
- t1.close();
-
- HTable t2 = (HTable) conn.getTable(table2);
- Put p2;
- for (int i = 0; i < 5; i++) {
- p2 = new Put(Bytes.toBytes("row-t2" + i));
- p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
- t2.put(p2);
- }
-
- Assert.assertThat(TEST_UTIL.countRows(t2), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5));
- t2.close();
-
- // #3 - incremental backup for multiple tables
- String tablesetIncMultiple =
- table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString();
-
- String backupIdIncMultiple = BackupClient.create("incremental", BACKUP_ROOT_DIR,
- tablesetIncMultiple, null);
- assertTrue(checkSucceeded(backupIdIncMultiple));
-
-
- // #4 - restore full backup for all tables, without overwrite
- String[] tablesRestoreFull =
- new String[] { table1.getNameAsString(), table2.getNameAsString(),
- table3.getNameAsString(), table4.getNameAsString() };
-
- String[] tablesMapFull =
- new String[] { table1_restore, table2_restore, table3_restore, table4_restore };
-
- hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdFull);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdFull, false, false,
- tablesRestoreFull,
- tablesMapFull, false);
-
- // #5.1 - check tables for full restore
- HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
- assertTrue(hAdmin.tableExists(TableName.valueOf(table1_restore)));
- assertTrue(hAdmin.tableExists(TableName.valueOf(table2_restore)));
- assertTrue(hAdmin.tableExists(TableName.valueOf(table3_restore)));
- assertTrue(hAdmin.tableExists(TableName.valueOf(table4_restore)));
-
- hAdmin.close();
-
- // #5.2 - checking row count of tables for full restore
- HTable hTable = (HTable) conn.getTable(TableName.valueOf(table1_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
- hTable.close();
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table2_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
- hTable.close();
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table3_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
- hTable.close();
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table4_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
- hTable.close();
-
- // #6 - restore incremental backup for multiple tables, with overwrite
- String[] tablesRestoreIncMultiple =
- new String[]
- { table1.getNameAsString(), table2.getNameAsString(), table3.getNameAsString() };
- String[] tablesMapIncMultiple =
- new String[] { table1_restore, table2_restore, table3_restore };
- hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdIncMultiple, false, false,
- tablesRestoreIncMultiple, tablesMapIncMultiple, true);
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table1_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2));
- hTable.close();
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table2_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5));
- hTable.close();
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table3_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
- hTable.close();
-
- // #7 - incremental backup for single, empty table
-
- String tablesetIncEmpty = table4.getNameAsString();
- String backupIdIncEmpty =
- BackupClient.create("incremental", BACKUP_ROOT_DIR, tablesetIncEmpty, null);
- assertTrue(checkSucceeded(backupIdIncEmpty));
-
-
- // #8 - restore incremental backup for single empty table, with overwrite
- String[] tablesRestoreIncEmpty = new String[] { table4.getNameAsString() };
- String[] tablesMapIncEmpty = new String[] { table4_restore };
- hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncEmpty);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdIncEmpty, false, false,
- tablesRestoreIncEmpty,
- tablesMapIncEmpty, true);
-
- hTable = (HTable) conn.getTable(TableName.valueOf(table4_restore));
- Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0));
- hTable.close();
- conn.close();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
deleted file mode 100644
index 03822a2..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestRemoteBackup extends TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestRemoteBackup.class);
-
- /**
- * Verify that a remote full backup is created on a single table with data correctly.
- * @throws Exception
- */
- @Test
- public void testFullBackupRemote() throws Exception {
-
- LOG.info("test remote full backup on a single table");
-
- // String rootdir = TEST_UTIL2.getDefaultRootDirPath() + BACKUP_ROOT_DIR;
- // LOG.info("ROOTDIR " + rootdir);
- String backupId =
- BackupClient.create("full", BACKUP_REMOTE_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
deleted file mode 100644
index e1315c5..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestRemoteRestore extends TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestRemoteRestore.class);
-
- /**
- * Verify that a remote restore on a single table is successful.
- * @throws Exception
- */
- @Test
- public void testFullRestoreRemote() throws Exception {
-
- LOG.info("test remote full backup on a single table");
- String backupId =
- BackupClient.create("full", BACKUP_REMOTE_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
- String[] tableset = new String[] { table1.getNameAsString() };
- String[] tablemap = new String[] { table1_restore };
- Path path = new Path(BACKUP_REMOTE_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_REMOTE_ROOT_DIR, backupId, false, false, tableset,
- tablemap, false);
- HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
- assertTrue(hba.tableExists(TableName.valueOf(table1_restore)));
- TEST_UTIL.deleteTable(TableName.valueOf(table1_restore));
- hba.close();
- }
-
-}
\ No newline at end of file
[23/29] hbase git commit: HBASE-15044 Region normalization should be
allowed when underlying namespace has quota
Posted by sy...@apache.org.
HBASE-15044 Region normalization should be allowed when underlying namespace has quota
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1b6d47e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1b6d47e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1b6d47e
Branch: refs/heads/hbase-12439
Commit: c1b6d47e7974a5d9d75933bab9a28572e9d95c14
Parents: 07b6236
Author: tedyu <yu...@gmail.com>
Authored: Wed Dec 30 08:04:23 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Wed Dec 30 08:04:23 2015 -0800
----------------------------------------------------------------------
.../hadoop/hbase/master/AssignmentManager.java | 19 ++++++--
.../org/apache/hadoop/hbase/master/HMaster.java | 14 +++---
.../hadoop/hbase/master/MasterServices.java | 6 +++
.../normalizer/EmptyNormalizationPlan.java | 6 +++
.../normalizer/MergeNormalizationPlan.java | 6 +++
.../master/normalizer/NormalizationPlan.java | 10 ++++
.../master/normalizer/RegionNormalizer.java | 16 +++++++
.../normalizer/SimpleRegionNormalizer.java | 12 +++++
.../normalizer/SplitNormalizationPlan.java | 6 +++
.../hadoop/hbase/master/TestCatalogJanitor.java | 6 +++
.../TestSimpleRegionNormalizerOnCluster.java | 50 +++++++++++++++-----
.../hbase/namespace/TestNamespaceAuditor.java | 10 ++--
12 files changed, 135 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index f7f98fe..4feb2e7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -72,8 +72,10 @@ import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.quotas.QuotaExceededException;
import org.apache.hadoop.hbase.quotas.RegionStateListener;
import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
import org.apache.hadoop.hbase.regionserver.RegionServerAbortedException;
@@ -1654,9 +1656,13 @@ public class AssignmentManager {
}
}
RegionLocations rl = MetaTableAccessor.getRegionLocations(result);
- if (rl == null) continue;
+ if (rl == null) {
+ continue;
+ }
HRegionLocation[] locations = rl.getRegionLocations();
- if (locations == null) continue;
+ if (locations == null) {
+ continue;
+ }
for (HRegionLocation hrl : locations) {
if (hrl == null) continue;
HRegionInfo regionInfo = hrl.getRegionInfo();
@@ -1996,7 +2002,9 @@ public class AssignmentManager {
*/
public boolean waitOnRegionToClearRegionsInTransition(final HRegionInfo hri, long timeOut)
throws InterruptedException {
- if (!regionStates.isRegionInTransition(hri)) return true;
+ if (!regionStates.isRegionInTransition(hri)) {
+ return true;
+ }
long end = (timeOut <= 0) ? Long.MAX_VALUE : EnvironmentEdgeManager.currentTime()
+ timeOut;
// There is already a timeout monitor on regions in transition so I
@@ -2836,7 +2844,10 @@ public class AssignmentManager {
regionStateListener.onRegionSplit(hri);
errorMsg = onRegionReadyToSplit(current, hri, serverName, transition);
} catch (IOException exp) {
- errorMsg = StringUtils.stringifyException(exp);
+ if (exp instanceof QuotaExceededException) {
+ server.getRegionNormalizer().planSkipped(hri, PlanType.SPLIT);
+ }
+ errorMsg = StringUtils.stringifyException(exp);
}
break;
case SPLIT_PONR:
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bdb19f4..8e51f25 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory;
@@ -1326,6 +1327,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
return true;
}
+ @VisibleForTesting
+ public RegionNormalizer getRegionNormalizer() {
+ return this.normalizer;
+ }
+
/**
* Perform normalization of cluster (invoked by {@link RegionNormalizerChore}).
*
@@ -1353,11 +1359,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
Collections.shuffle(allEnabledTables);
for (TableName table : allEnabledTables) {
- if (quotaManager.getNamespaceQuotaManager() != null &&
- quotaManager.getNamespaceQuotaManager().getState(table.getNamespaceAsString()) != null){
- LOG.debug("Skipping normalizing " + table + " since its namespace has quota");
- continue;
- }
TableDescriptor tblDesc = getTableDescriptors().getDescriptor(table);
if (table.isSystemTable() || (tblDesc != null &&
tblDesc.getHTableDescriptor() != null &&
@@ -1366,7 +1367,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
+ " table or doesn't have auto normalization turned on");
continue;
}
- this.normalizer.computePlanForTable(table).execute(clusterConnection.getAdmin());
+ NormalizationPlan plan = this.normalizer.computePlanForTable(table);
+ plan.execute(clusterConnection.getAdmin());
}
}
// If Region did not generate any plans, it means the cluster is already balanced.
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 935644f..af0e490 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.executor.ExecutorService;
@@ -88,6 +89,11 @@ public interface MasterServices extends Server {
* @return Master's instance of {@link MasterQuotaManager}
*/
MasterQuotaManager getMasterQuotaManager();
+
+ /**
+ * @return Master's instance of {@link RegionNormalizer}
+ */
+ RegionNormalizer getRegionNormalizer();
/**
* @return Master's instance of {@link ProcedureExecutor}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java
index a36dd07..5aecc48 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.normalizer;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
/**
* Plan which signifies that no normalization is required,
@@ -45,4 +46,9 @@ public final class EmptyNormalizationPlan implements NormalizationPlan {
@Override
public void execute(Admin admin) {
}
+
+ @Override
+ public PlanType getType() {
+ return PlanType.NONE;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java
index 08a58a5..e2035bb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
import java.io.IOException;
@@ -41,6 +42,11 @@ public class MergeNormalizationPlan implements NormalizationPlan {
this.secondRegion = secondRegion;
}
+ @Override
+ public PlanType getType() {
+ return PlanType.MERGE;
+ }
+
HRegionInfo getFirstRegion() {
return firstRegion;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java
index 96eed8c..9f866d3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/NormalizationPlan.java
@@ -26,10 +26,20 @@ import org.apache.hadoop.hbase.client.Admin;
*/
@InterfaceAudience.Private
public interface NormalizationPlan {
+ enum PlanType {
+ SPLIT,
+ MERGE,
+ NONE
+ }
/**
* Executes normalization plan on cluster (does actual splitting/merging work).
* @param admin instance of Admin
*/
void execute(Admin admin);
+
+ /**
+ * @return the type of this plan
+ */
+ PlanType getType();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
index 19abcf2..d60474d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.hbase.master.normalizer;
import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
/**
* Performs "normalization" of regions on the cluster, making sure that suboptimal
@@ -48,4 +50,18 @@ public interface RegionNormalizer {
* @return Next (perhaps most urgent) normalization action to perform
*/
NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException;
+
+ /**
+ * Notification for the case where plan couldn't be executed due to constraint violation, such as
+ * namespace quota
+ * @param hri the region which is involved in the plan
+ * @param type type of plan
+ */
+ void planSkipped(HRegionInfo hri, PlanType type);
+
+ /**
+ * @param type type of plan for which skipped count is to be returned
+ * @return the count of plans of specified type which were skipped
+ */
+ long getSkippedCount(NormalizationPlan.PlanType type);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index fe6034b..fe10bd1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
import org.apache.hadoop.hbase.util.Triple;
import java.util.ArrayList;
@@ -59,6 +60,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class);
private static final int MIN_REGION_COUNT = 3;
private MasterServices masterServices;
+ private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
/**
* Set the master service.
@@ -85,6 +87,16 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
}
};
+ @Override
+ public void planSkipped(HRegionInfo hri, PlanType type) {
+ skippedCount[type.ordinal()]++;
+ }
+
+ @Override
+ public long getSkippedCount(NormalizationPlan.PlanType type) {
+ return skippedCount[type.ordinal()];
+ }
+
/**
* Computes next most "urgent" normalization action on the table.
* Action may be either a split, or a merge, or no action.
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java
index c96988a..b95bfb7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
import java.io.IOException;
import java.util.Arrays;
@@ -42,6 +43,11 @@ public class SplitNormalizationPlan implements NormalizationPlan {
this.splitPoint = splitPoint;
}
+ @Override
+ public PlanType getType() {
+ return PlanType.SPLIT;
+ }
+
public HRegionInfo getRegionInfo() {
return regionInfo;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index b5f20f5..8e35bbf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLog
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
+import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -252,6 +253,11 @@ public class TestCatalogJanitor {
}
@Override
+ public RegionNormalizer getRegionNormalizer() {
+ return null;
+ }
+
+ @Override
public MasterFileSystem getMasterFileSystem() {
return this.mfs;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
index d24023d..4fe42ed 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java
@@ -25,11 +25,16 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.TableNamespaceManager;
+import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
+import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor;
+import org.apache.hadoop.hbase.quotas.QuotaUtil;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -63,9 +68,11 @@ public class TestSimpleRegionNormalizerOnCluster {
// we will retry operations when PleaseHoldException is thrown
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
TEST_UTIL.getConfiguration().setBoolean(HConstants.HBASE_NORMALIZER_ENABLED, true);
+ TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
// Start a cluster of two regionservers.
TEST_UTIL.startMiniCluster(1);
+ TestNamespaceAuditor.waitForQuotaEnabled(TEST_UTIL);
admin = TEST_UTIL.getHBaseAdmin();
}
@@ -74,11 +81,27 @@ public class TestSimpleRegionNormalizerOnCluster {
TEST_UTIL.shutdownMiniCluster();
}
- @Test(timeout = 60000)
+ @Test(timeout = 90000)
@SuppressWarnings("deprecation")
public void testRegionNormalizationSplitOnCluster() throws Exception {
- final TableName TABLENAME =
- TableName.valueOf("testRegionNormalizationSplitOnCluster");
+ testRegionNormalizationSplitOnCluster(false);
+ testRegionNormalizationSplitOnCluster(true);
+ }
+
+ void testRegionNormalizationSplitOnCluster(boolean limitedByQuota) throws Exception {
+ TableName TABLENAME;
+ if (limitedByQuota) {
+ String nsp = "np2";
+ NamespaceDescriptor nspDesc =
+ NamespaceDescriptor.create(nsp)
+ .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
+ .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
+ admin.createNamespace(nspDesc);
+ TABLENAME = TableName.valueOf(nsp +
+ TableName.NAMESPACE_DELIM + "testRegionNormalizationSplitOnCluster");
+ } else {
+ TABLENAME = TableName.valueOf("testRegionNormalizationSplitOnCluster");
+ }
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster m = cluster.getMaster();
@@ -119,21 +142,26 @@ public class TestSimpleRegionNormalizerOnCluster {
admin.flush(TABLENAME);
- System.out.println(admin.getTableDescriptor(TABLENAME));
-
assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
// Now trigger a split and stop when the split is in progress
Thread.sleep(5000); // to let region load to update
m.normalizeRegions();
-
- while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) {
- LOG.info("Waiting for normalization split to complete");
- Thread.sleep(100);
+ if (limitedByQuota) {
+ long skippedSplitcnt = 0;
+ do {
+ skippedSplitcnt = m.getRegionNormalizer().getSkippedCount(PlanType.SPLIT);
+ Thread.sleep(100);
+ } while (skippedSplitcnt == 0L);
+ assert(skippedSplitcnt > 0);
+ } else {
+ while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) {
+ LOG.info("Waiting for normalization split to complete");
+ Thread.sleep(100);
+ }
+ assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
}
- assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME));
-
admin.disableTable(TABLENAME);
admin.deleteTable(TABLENAME);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1b6d47e/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 7ded3d3..e3b841b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -115,7 +115,7 @@ public class TestNamespaceAuditor {
conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
RegionServerObserver.class);
UTIL.startMiniCluster(1, 1);
- waitForQuotaEnabled();
+ waitForQuotaEnabled(UTIL);
ADMIN = UTIL.getHBaseAdmin();
}
@@ -620,11 +620,11 @@ public class TestNamespaceAuditor {
.getTables().size(), after.getTables().size());
}
- private static void waitForQuotaEnabled() throws Exception {
- UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
+ public static void waitForQuotaEnabled(final HBaseTestingUtility util) throws Exception {
+ util.waitFor(60000, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
- HMaster master = UTIL.getHBaseCluster().getMaster();
+ HMaster master = util.getHBaseCluster().getMaster();
if (master == null) {
return false;
}
@@ -638,7 +638,7 @@ public class TestNamespaceAuditor {
UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
UTIL.getHBaseCluster().waitOnMaster(0);
UTIL.getHBaseCluster().startMaster();
- waitForQuotaEnabled();
+ waitForQuotaEnabled(UTIL);
}
private NamespaceAuditor getQuotaManager() {
[28/29] hbase git commit: HBASE-15038 ExportSnapshot should support
separate configs for source and destination
Posted by sy...@apache.org.
HBASE-15038 ExportSnapshot should support separate configs for source and destination
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9589a7d8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9589a7d8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9589a7d8
Branch: refs/heads/hbase-12439
Commit: 9589a7d8be4d29bffcb0c711e5bd6573e8df712c
Parents: a82f7fc
Author: Gary Helmling <ga...@apache.org>
Authored: Wed Dec 23 18:49:58 2015 -0800
Committer: Gary Helmling <ga...@apache.org>
Committed: Mon Jan 4 00:10:19 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/snapshot/ExportSnapshot.java | 33 ++++++++++++++------
1 file changed, 23 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9589a7d8/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index beddce3..a574410 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -89,6 +89,10 @@ import org.apache.hadoop.util.ToolRunner;
@InterfaceStability.Evolving
public class ExportSnapshot extends Configured implements Tool {
public static final String NAME = "exportsnapshot";
+ /** Configuration prefix for overrides for the source filesystem */
+ public static final String CONF_SOURCE_PREFIX = NAME + ".from.";
+ /** Configuration prefix for overrides for the destination filesystem */
+ public static final String CONF_DEST_PREFIX = NAME + ".to.";
private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
@@ -141,6 +145,9 @@ public class ExportSnapshot extends Configured implements Tool {
@Override
public void setup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
+ Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
+ Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
+
verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
filesGroup = conf.get(CONF_FILES_GROUP);
@@ -155,15 +162,15 @@ public class ExportSnapshot extends Configured implements Tool {
testFailures = conf.getBoolean(CONF_TEST_FAILURE, false);
try {
- conf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
- inputFs = FileSystem.get(inputRoot.toUri(), conf);
+ srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
+ inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
} catch (IOException e) {
throw new IOException("Could not get the input FileSystem with root=" + inputRoot, e);
}
try {
- conf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
- outputFs = FileSystem.get(outputRoot.toUri(), conf);
+ destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
+ outputFs = FileSystem.get(outputRoot.toUri(), destConf);
} catch (IOException e) {
throw new IOException("Could not get the output FileSystem with root="+ outputRoot, e);
}
@@ -789,8 +796,12 @@ public class ExportSnapshot extends Configured implements Tool {
job.setNumReduceTasks(0);
// Acquire the delegation Tokens
+ Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
+ TokenCache.obtainTokensForNamenodes(job.getCredentials(),
+ new Path[] { inputRoot }, srcConf);
+ Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
- new Path[] { inputRoot, outputRoot }, conf);
+ new Path[] { outputRoot }, destConf);
// Run the MR Job
if (!job.waitForCompletion(true)) {
@@ -913,11 +924,13 @@ public class ExportSnapshot extends Configured implements Tool {
targetName = snapshotName;
}
- conf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
- FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf);
+ Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
+ srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
+ FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
- conf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
- FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
+ Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
+ destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
+ FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf);
LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
@@ -1004,7 +1017,7 @@ public class ExportSnapshot extends Configured implements Tool {
// Step 4 - Verify snapshot integrity
if (verifyTarget) {
LOG.info("Verify snapshot integrity");
- verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir);
+ verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);
}
LOG.info("Export Completed: " + targetName);
[10/29] hbase git commit: HBASE-14030 Revert due to pending review
comments
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java
deleted file mode 100644
index 14769f9..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTable.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
-import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-
-/**
- * This class provides 'hbase:backup' table API
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class BackupSystemTable {
-
- private static final Log LOG = LogFactory.getLog(BackupSystemTable.class);
- private final static String TABLE_NAMESPACE = "hbase";
- private final static String TABLE_NAME = "backup";
- private final static TableName tableName = TableName.valueOf(TABLE_NAMESPACE, TABLE_NAME);
- public final static byte[] familyName = "f".getBytes();
-
- // Connection to HBase cluster
- private static Connection connection;
- // Cluster configuration
- private static Configuration config;
- // singleton
- private static BackupSystemTable table;
-
- /**
- * Get instance by a given configuration
- * @param conf - HBase configuration
- * @return instance of BackupSystemTable
- * @throws IOException exception
- */
- public synchronized static BackupSystemTable getTable(Configuration conf) throws IOException {
- if (connection == null) {
- connection = ConnectionFactory.createConnection(conf);
- config = conf;
- // Verify hbase:system exists
- createSystemTableIfNotExists();
- table = new BackupSystemTable();
- }
- return table;
- }
-
- /**
- * TODO: refactor
- * @throws IOException exception
- */
- public static void close() throws IOException {
- connection.close();
- table = null;
- }
-
- /**
- * Gets table name
- * @return table name
- */
- public static TableName getTableName() {
- return tableName;
- }
-
- private static void createSystemTableIfNotExists() throws IOException {
- Admin admin = null;
- try {
- admin = connection.getAdmin();
- if (admin.tableExists(tableName) == false) {
- HTableDescriptor tableDesc = new HTableDescriptor(tableName);
- HColumnDescriptor colDesc = new HColumnDescriptor(familyName);
- colDesc.setMaxVersions(1);
- int ttl =
- config.getInt(HConstants.BACKUP_SYSTEM_TTL_KEY, HConstants.BACKUP_SYSTEM_TTL_DEFAULT);
- colDesc.setTimeToLive(ttl);
- tableDesc.addFamily(colDesc);
- admin.createTable(tableDesc);
- }
- } catch (IOException e) {
- LOG.error(e);
- throw e;
- } finally {
- if (admin != null) {
- admin.close();
- }
- }
- }
-
- private BackupSystemTable() {
- }
-
- /**
- * Updates status (state) of a backup session in hbase:backup table
- * @param context context
- * @throws IOException exception
- */
- public void updateBackupStatus(BackupContext context) throws IOException {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("update backup status in hbase:backup for: " + context.getBackupId()
- + " set status=" + context.getFlag());
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Put put = BackupSystemTableHelper.createPutForBackupContext(context);
- table.put(put);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Deletes backup status from hbase:backup table
- * @param backupId backup id
- * @throws IOException exception
- */
-
- public void deleteBackupStatus(String backupId) throws IOException {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("delete backup status in hbase:backup for " + backupId);
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Delete del = BackupSystemTableHelper.createDeletForBackupContext(backupId);
- table.delete(del);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Reads backup status object (instance of BackupContext) from hbase:backup table
- * @param backupId - backupId
- * @return Current status of backup session or null
- */
-
- public BackupContext readBackupStatus(String backupId) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("read backup status from hbase:backup for: " + backupId);
- }
-
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Get get = BackupSystemTableHelper.createGetForBackupContext(backupId);
- Result res = table.get(get);
- if(res.isEmpty()){
- return null;
- }
- return BackupSystemTableHelper.resultToBackupContext(res);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Read the last backup start code (timestamp) of last successful backup. Will return null if
- * there is no start code stored on hbase or the value is of length 0. These two cases indicate
- * there is no successful backup completed so far.
- * @return the timestamp of last successful backup
- * @throws IOException exception
- */
- public String readBackupStartCode() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("read backup start code from hbase:backup");
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Get get = BackupSystemTableHelper.createGetForStartCode();
- Result res = table.get(get);
- if (res.isEmpty()){
- return null;
- }
- Cell cell = res.listCells().get(0);
- byte[] val = CellUtil.cloneValue(cell);
- if (val.length == 0){
- return null;
- }
- return new String(val);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte.
- * @param startCode start code
- * @throws IOException exception
- */
- public void writeBackupStartCode(String startCode) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("write backup start code to hbase:backup " + startCode);
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Put put = BackupSystemTableHelper.createPutForStartCode(startCode);
- table.put(put);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Get the Region Servers log information after the last log roll from hbase:backup.
- * @return RS log info
- * @throws IOException exception
- */
- public HashMap<String, String> readRegionServerLastLogRollResult()
- throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("read region server last roll log result to hbase:backup");
- }
- Table table = null;
- ResultScanner scanner = null;
-
- try {
- table = connection.getTable(tableName);
- Scan scan = BackupSystemTableHelper.createScanForReadRegionServerLastLogRollResult();
- scan.setMaxVersions(1);
- scanner = table.getScanner(scan);
- Result res = null;
- HashMap<String, String> rsTimestampMap = new HashMap<String, String>();
- while ((res = scanner.next()) != null) {
- res.advance();
- Cell cell = res.current();
- byte[] row = CellUtil.cloneRow(cell);
- String server =
- BackupSystemTableHelper.getServerNameForReadRegionServerLastLogRollResult(row);
-
- byte[] data = CellUtil.cloneValue(cell);
- rsTimestampMap.put(server, new String(data));
- }
- return rsTimestampMap;
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Writes Region Server last roll log result (timestamp) to hbase:backup table
- * @param server - Region Server name
- * @param fileName - last log timestamp
- * @throws IOException exception
- */
- public void writeRegionServerLastLogRollResult(String server, String fileName)
- throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("write region server last roll log result to hbase:backup");
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Put put =
- BackupSystemTableHelper.createPutForRegionServerLastLogRollResult(server, fileName);
- table.put(put);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Get all completed backup information (in desc order by time)
- * @return history info of BackupCompleteData
- * @throws IOException exception
- */
- public ArrayList<BackupCompleteData> getBackupHistory() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("get backup history from hbase:backup");
- }
- Table table = null;
- ResultScanner scanner = null;
- ArrayList<BackupCompleteData> list = new ArrayList<BackupCompleteData>();
- try {
- table = connection.getTable(tableName);
- Scan scan = BackupSystemTableHelper.createScanForBackupHistory();
- scan.setMaxVersions(1);
- scanner = table.getScanner(scan);
- Result res = null;
- while ((res = scanner.next()) != null) {
- res.advance();
- BackupContext context = BackupSystemTableHelper.cellToBackupContext(res.current());
- if (context.getFlag() != BACKUPSTATUS.COMPLETE) {
- continue;
- }
-
- BackupCompleteData history = new BackupCompleteData();
- history.setBackupToken(context.getBackupId());
- history.setStartTime(Long.toString(context.getStartTs()));
- history.setEndTime(Long.toString(context.getEndTs()));
- history.setBackupRootPath(context.getTargetRootDir());
- history.setTableList(context.getTableListAsString());
- history.setType(context.getType());
- history.setBytesCopied(Long.toString(context.getTotalBytesCopied()));
-
- if (context.fromExistingSnapshot()) {
- history.markFromExistingSnapshot();
- }
- list.add(history);
- }
- return BackupUtil.sortHistoryListDesc(list);
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Get all backup session with a given status (in desc order by time)
- * @param status status
- * @return history info of backup contexts
- * @throws IOException exception
- */
- public ArrayList<BackupContext> getBackupContexts(BACKUPSTATUS status) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("get backup contexts from hbase:backup");
- }
- Table table = null;
- ResultScanner scanner = null;
- ArrayList<BackupContext> list = new ArrayList<BackupContext>();
- try {
- table = connection.getTable(tableName);
- Scan scan = BackupSystemTableHelper.createScanForBackupHistory();
- scan.setMaxVersions(1);
- scanner = table.getScanner(scan);
- Result res = null;
- while ((res = scanner.next()) != null) {
- res.advance();
- BackupContext context = BackupSystemTableHelper.cellToBackupContext(res.current());
- if (context.getFlag() != status){
- continue;
- }
- list.add(context);
- }
- return list;
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Write the current timestamps for each regionserver to hbase:backup after a successful full or
- * incremental backup. The saved timestamp is of the last log file that was backed up already.
- * @param tables tables
- * @param newTimestamps timestamps
- * @throws IOException exception
- */
- public void writeRegionServerLogTimestamp(Set<String> tables,
- HashMap<String, String> newTimestamps) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("write RS log ts to HBASE_BACKUP");
- }
- StringBuilder sb = new StringBuilder();
- for (Map.Entry<String, String> entry : newTimestamps.entrySet()) {
- String host = entry.getKey();
- String timestamp = entry.getValue();
- sb.append(host).append(BackupUtil.FIELD_SEPARATOR).append(timestamp)
- .append(BackupUtil.RECORD_SEPARATOR);
- }
- String smap = sb.toString();
- List<Put> puts = new ArrayList<Put>();
- for (String table : tables) {
- Put put = BackupSystemTableHelper.createPutForWriteRegionServerLogTimestamp(table, smap);
- puts.add(put);
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- table.put(puts);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Read the timestamp for each region server log after the last successful backup. Each table has
- * its own set of the timestamps. The info is stored for each table as a concatenated string of
- * rs->timestapmp
- * @return the timestamp for each region server. key: tableName value:
- * RegionServer,PreviousTimeStamp
- * @throws IOException exception
- */
- public HashMap<String, HashMap<String, String>> readLogTimestampMap() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("read RS log ts from HBASE_BACKUP");
- }
-
- Table table = null;
- ResultScanner scanner = null;
- HashMap<String, HashMap<String, String>> tableTimestampMap =
- new HashMap<String, HashMap<String, String>>();
-
- try {
- table = connection.getTable(tableName);
- Scan scan = BackupSystemTableHelper.createScanForReadLogTimestampMap();
- scanner = table.getScanner(scan);
- Result res = null;
- while ((res = scanner.next()) != null) {
- res.advance();
- Cell cell = res.current();
- byte[] row = CellUtil.cloneRow(cell);
- String tabName = BackupSystemTableHelper.getTableNameForReadLogTimestampMap(row);
- HashMap<String, String> lastBackup = new HashMap<String, String>();
- byte[] data = CellUtil.cloneValue(cell);
- if (data == null) {
- // TODO
- throw new IOException("Data of last backup data from HBASE_BACKUP "
- + "is empty. Create a backup first.");
- }
- if (data != null && data.length > 0) {
- String s = new String(data);
- String[] records = s.split(BackupUtil.RECORD_SEPARATOR);
- for (String record : records) {
- String[] flds = record.split(BackupUtil.FIELD_SEPARATOR);
- if (flds.length != 2) {
- throw new IOException("data from HBASE_BACKUP is corrupted: "
- + Arrays.toString(flds));
- }
- lastBackup.put(flds[0], flds[1]);
- }
- tableTimestampMap.put(tabName, lastBackup);
- }
- }
- return tableTimestampMap;
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Return the current tables covered by incremental backup.
- * @return set of tableNames
- * @throws IOException exception
- */
- public Set<String> getIncrementalBackupTableSet() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("get incr backup table set from hbase:backup");
- }
- Table table = null;
- TreeSet<String> set = new TreeSet<String>();
-
- try {
- table = connection.getTable(tableName);
- Get get = BackupSystemTableHelper.createGetForIncrBackupTableSet();
- Result res = table.get(get);
- if (res.isEmpty()) {
- return set;
- }
- List<Cell> cells = res.listCells();
- for (Cell cell : cells) {
- // qualifier = table name - we use table names as qualifiers
- // TODO ns:table as qualifier?
- set.add(new String(CellUtil.cloneQualifier(cell)));
- }
- return set;
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Add tables to global incremental backup set
- * @param tables - set of tables
- * @throws IOException exception
- */
- public void addIncrementalBackupTableSet(Set<String> tables) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("add incr backup table set to hbase:backup");
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Put put = BackupSystemTableHelper.createPutForIncrBackupTableSet(tables);
- table.put(put);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Register WAL files as eligible for deletion
- * @param files files
- * @throws IOException exception
- */
- public void addWALFiles(List<String> files, String backupId) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("add WAL files to hbase:backup");
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- List<Put> puts = BackupSystemTableHelper.createPutsForAddWALFiles(files, backupId);
- table.put(puts);
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Check if WAL file is eligible for deletion
- * @param file file
- * @return true, if - yes.
- * @throws IOException exception
- */
- public boolean checkWALFile(String file) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Check if WAL file has been already backuped in hbase:backup");
- }
- Table table = null;
- try {
- table = connection.getTable(tableName);
- Get get = BackupSystemTableHelper.createGetForCheckWALFile(file);
- Result res = table.get(get);
- if (res.isEmpty()){
- return false;
- }
- return true;
- } finally {
- if (table != null) {
- table.close();
- }
- }
- }
-
- /**
- * Checks if we have at least one backup session in hbase:backup This API is used by
- * BackupLogCleaner
- * @return true, if - at least one session exists in hbase:backup table
- * @throws IOException exception
- */
- public boolean hasBackupSessions() throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("has backup sessions from hbase:backup");
- }
- Table table = null;
- ResultScanner scanner = null;
- boolean result = false;
- try {
- table = connection.getTable(tableName);
- Scan scan = BackupSystemTableHelper.createScanForBackupHistory();
- scan.setMaxVersions(1);
- scan.setCaching(1);
- scanner = table.getScanner(scan);
- if (scanner.next() != null) {
- result = true;
- }
- return result;
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- if (table != null) {
- table.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java
deleted file mode 100644
index bf62a84..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupSystemTableHelper.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-
-
-/**
- * A collection for methods used by BackupSystemTable.
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class BackupSystemTableHelper {
-
- /**
- * hbase:backup schema:
- * 1. Backup sessions rowkey= "session." + backupId; value = serialized
- * BackupContext
- * 2. Backup start code rowkey = "startcode"; value = startcode
- * 3. Incremental backup set rowkey="incrbackupset"; value=[list of tables]
- * 4. Table-RS-timestamp map rowkey="trslm"+ table_name; value = map[RS-> last WAL timestamp]
- * 5. RS - WAL ts map rowkey="rslogts."+server; value = last WAL timestamp
- * 6. WALs recorded rowkey="wals."+WAL unique file name; value = NULL (value is not used)
- */
- private static final Log LOG = LogFactory.getLog(BackupSystemTableHelper.class);
-
- private final static String BACKUP_CONTEXT_PREFIX = "session.";
- private final static String START_CODE_ROW = "startcode";
- private final static String INCR_BACKUP_SET = "incrbackupset";
- private final static String TABLE_RS_LOG_MAP_PREFIX = "trslm.";
- private final static String RS_LOG_TS_PREFIX = "rslogts.";
- private final static String WALS_PREFIX = "wals.";
-
- private final static byte[] q0 = "0".getBytes();
- private final static byte[] EMPTY_VALUE = new byte[] {};
-
- private BackupSystemTableHelper() {
- throw new AssertionError("Instantiating utility class...");
- }
-
- /**
- * Creates Put operation for a given backup context object
- * @param context backup context
- * @return put operation
- * @throws IOException exception
- */
- static Put createPutForBackupContext(BackupContext context) throws IOException {
-
- Put put = new Put((BACKUP_CONTEXT_PREFIX + context.getBackupId()).getBytes());
- put.addColumn(BackupSystemTable.familyName, q0, context.toByteArray());
- return put;
- }
-
- /**
- * Creates Get operation for a given backup id
- * @param backupId - backup's ID
- * @return get operation
- * @throws IOException exception
- */
- static Get createGetForBackupContext(String backupId) throws IOException {
- Get get = new Get((BACKUP_CONTEXT_PREFIX + backupId).getBytes());
- get.addFamily(BackupSystemTable.familyName);
- get.setMaxVersions(1);
- return get;
- }
-
- /**
- * Creates Delete operation for a given backup id
- * @param backupId - backup's ID
- * @return delete operation
- * @throws IOException exception
- */
- public static Delete createDeletForBackupContext(String backupId) {
- Delete del = new Delete((BACKUP_CONTEXT_PREFIX + backupId).getBytes());
- del.addFamily(BackupSystemTable.familyName);
- return del;
- }
-
- /**
- * Converts Result to BackupContext
- * @param res - HBase result
- * @return backup context instance
- * @throws IOException exception
- */
- static BackupContext resultToBackupContext(Result res) throws IOException {
- res.advance();
- Cell cell = res.current();
- return cellToBackupContext(cell);
- }
-
- /**
- * Creates Get operation to retrieve start code from hbase:backup
- * @return get operation
- * @throws IOException exception
- */
- static Get createGetForStartCode() throws IOException {
- Get get = new Get(START_CODE_ROW.getBytes());
- get.addFamily(BackupSystemTable.familyName);
- get.setMaxVersions(1);
- return get;
- }
-
- /**
- * Creates Put operation to store start code to hbase:backup
- * @return put operation
- * @throws IOException exception
- */
- static Put createPutForStartCode(String startCode) {
- Put put = new Put(START_CODE_ROW.getBytes());
- put.addColumn(BackupSystemTable.familyName, q0, startCode.getBytes());
- return put;
- }
-
- /**
- * Creates Get to retrieve incremental backup table set from hbase:backup
- * @return get operation
- * @throws IOException exception
- */
- static Get createGetForIncrBackupTableSet() throws IOException {
- Get get = new Get(INCR_BACKUP_SET.getBytes());
- get.addFamily(BackupSystemTable.familyName);
- get.setMaxVersions(1);
- return get;
- }
-
- /**
- * Creates Put to store incremental backup table set
- * @param tables tables
- * @return put operation
- */
- static Put createPutForIncrBackupTableSet(Set<String> tables) {
- Put put = new Put(INCR_BACKUP_SET.getBytes());
- for (String table : tables) {
- put.addColumn(BackupSystemTable.familyName, table.getBytes(), EMPTY_VALUE);
- }
- return put;
- }
-
- /**
- * Creates Scan operation to load backup history
- * @return scan operation
- */
- static Scan createScanForBackupHistory() {
- Scan scan = new Scan();
- byte[] startRow = BACKUP_CONTEXT_PREFIX.getBytes();
- byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
- stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
- scan.setStartRow(startRow);
- scan.setStopRow(stopRow);
- scan.addFamily(BackupSystemTable.familyName);
-
- return scan;
- }
-
- /**
- * Converts cell to backup context instance.
- * @param current - cell
- * @return backup context instance
- * @throws IOException exception
- */
- static BackupContext cellToBackupContext(Cell current) throws IOException {
- byte[] data = CellUtil.cloneValue(current);
- try {
- BackupContext ctxt = BackupContext.fromByteArray(data);
- return ctxt;
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Creates Put to write RS last roll log timestamp map
- * @param table - table
- * @param smap - map, containing RS:ts
- * @return put operation
- */
- static Put createPutForWriteRegionServerLogTimestamp(String table, String smap) {
- Put put = new Put((TABLE_RS_LOG_MAP_PREFIX + table).getBytes());
- put.addColumn(BackupSystemTable.familyName, q0, smap.getBytes());
- return put;
- }
-
- /**
- * Creates Scan to load table-> { RS -> ts} map of maps
- * @return scan operation
- */
- static Scan createScanForReadLogTimestampMap() {
- Scan scan = new Scan();
- byte[] startRow = TABLE_RS_LOG_MAP_PREFIX.getBytes();
- byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
- stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
- scan.setStartRow(startRow);
- scan.setStopRow(stopRow);
- scan.addFamily(BackupSystemTable.familyName);
-
- return scan;
- }
-
- /**
- * Get table name from rowkey
- * @param cloneRow rowkey
- * @return table name
- */
- static String getTableNameForReadLogTimestampMap(byte[] cloneRow) {
- int prefixSize = TABLE_RS_LOG_MAP_PREFIX.length();
- return new String(cloneRow, prefixSize, cloneRow.length - prefixSize);
- }
-
- /**
- * Creates Put to store RS last log result
- * @param server - server name
- * @param fileName - log roll result (timestamp)
- * @return put operation
- */
- static Put createPutForRegionServerLastLogRollResult(String server, String fileName) {
- Put put = new Put((RS_LOG_TS_PREFIX + server).getBytes());
- put.addColumn(BackupSystemTable.familyName, q0, fileName.getBytes());
- return put;
- }
-
- /**
- * Creates Scan operation to load last RS log roll results
- * @return scan operation
- */
- static Scan createScanForReadRegionServerLastLogRollResult() {
- Scan scan = new Scan();
- byte[] startRow = RS_LOG_TS_PREFIX.getBytes();
- byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
- stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
- scan.setStartRow(startRow);
- scan.setStopRow(stopRow);
- scan.addFamily(BackupSystemTable.familyName);
-
- return scan;
- }
-
- /**
- * Get server's name from rowkey
- * @param row - rowkey
- * @return server's name
- */
- static String getServerNameForReadRegionServerLastLogRollResult(byte[] row) {
- int prefixSize = RS_LOG_TS_PREFIX.length();
- return new String(row, prefixSize, row.length - prefixSize);
- }
-
- /**
- * Creates put list for list of WAL files
- * @param files list of WAL file paths
- * @param backupId backup id
- * @return put list
- * @throws IOException exception
- */
- public static List<Put> createPutsForAddWALFiles(List<String> files, String backupId)
- throws IOException {
-
- List<Put> puts = new ArrayList<Put>();
- for (String file : files) {
- LOG.debug("+++ put: " + BackupUtil.getUniqueWALFileNamePart(file));
- byte[] row = (WALS_PREFIX + BackupUtil.getUniqueWALFileNamePart(file)).getBytes();
- Put put = new Put(row);
- put.addColumn(BackupSystemTable.familyName, q0, backupId.getBytes());
- puts.add(put);
- }
- return puts;
- }
-
- /**
- * Creates Get operation for a given wal file name
- * @param file file
- * @return get operation
- * @throws IOException exception
- */
- public static Get createGetForCheckWALFile(String file) throws IOException {
- byte[] row = (WALS_PREFIX + BackupUtil.getUniqueWALFileNamePart(file)).getBytes();
- Get get = new Get(row);
- get.addFamily(BackupSystemTable.familyName);
- get.setMaxVersions(1);
- return get;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java
deleted file mode 100644
index ff8bd2e..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupUtil.java
+++ /dev/null
@@ -1,564 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.DefaultWALProvider;
-
-/**
- * A collection for methods used by multiple classes to backup HBase tables.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class BackupUtil {
- protected static final Log LOG = LogFactory.getLog(BackupUtil.class);
-
- public static final String FIELD_SEPARATOR = "\001";
- public static final String RECORD_SEPARATOR = "\002";
- public static final String LOGNAME_SEPARATOR = ".";
- protected static final String HDFS = "hdfs://";
- protected static Configuration conf = null;
-
- private BackupUtil(){
- throw new AssertionError("Instantiating utility class...");
- }
-
- /**
- * Set the configuration from a given one.
- * @param newConf A new given configuration
- */
- public synchronized static void setConf(Configuration newConf) {
- conf = newConf;
- }
-
- /**
- * Get and merge Hadoop and HBase configuration.
- * @throws IOException exception
- */
- protected static Configuration getConf() {
- if (conf == null) {
- conf = new Configuration();
- HBaseConfiguration.merge(conf, HBaseConfiguration.create());
- }
- return conf;
- }
-
- /**
- * Loop through the RS log timestamp map for the tables, for each RS, find the min timestamp value
- * for the RS among the tables.
- * @param rsLogTimestampMap timestamp map
- * @return the min timestamp of each RS
- */
- protected static HashMap<String, String> getRSLogTimestampMins(
- HashMap<String, HashMap<String, String>> rsLogTimestampMap) {
-
- if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) {
- return null;
- }
-
- HashMap<String, String> rsLogTimestamptMins = new HashMap<String, String>();
- HashMap<String, HashMap<String, String>> rsLogTimestampMapByRS =
- new HashMap<String, HashMap<String, String>>();
-
- for (Entry<String, HashMap<String, String>> tableEntry : rsLogTimestampMap.entrySet()) {
- String table = tableEntry.getKey();
- HashMap<String, String> rsLogTimestamp = tableEntry.getValue();
- for (Entry<String, String> rsEntry : rsLogTimestamp.entrySet()) {
- String rs = rsEntry.getKey();
- String ts = rsEntry.getValue();
- if (!rsLogTimestampMapByRS.containsKey(rs)) {
- rsLogTimestampMapByRS.put(rs, new HashMap<String, String>());
- rsLogTimestampMapByRS.get(rs).put(table, ts);
- } else {
- rsLogTimestampMapByRS.get(rs).put(table, ts);
- }
- }
- }
-
- for (String rs : rsLogTimestampMapByRS.keySet()) {
- rsLogTimestamptMins.put(rs, getMinValue(rsLogTimestampMapByRS.get(rs)));
- }
-
- return rsLogTimestamptMins;
- }
-
- /**
- * Get the min value for all the Values a map.
- * @param map map
- * @return the min value
- */
- protected static String getMinValue(HashMap<String, String> map) {
- String minTimestamp = null;
- if (map != null) {
- ArrayList<String> timestampList = new ArrayList<String>(map.values());
- Collections.sort(timestampList, new Comparator<String>() {
- @Override
- public int compare(String s1, String s2) {
- long l1 = Long.valueOf(s1);
- long l2 = Long.valueOf(s2);
- if (l1 > l2) {
- return 1;
- } else if (l1 < l2) {
- return -1;
- } else {
- return 0;
- }
- }
- });
- // The min among all the RS log timestamps will be kept in ZK.
- minTimestamp = timestampList.get(0);
- }
- return minTimestamp;
- }
-
- /**
- * copy out Table RegionInfo into incremental backup image need to consider move this logic into
- * HBackupFileSystem
- * @param backupContext backup context
- * @param conf configuration
- * @throws IOException exception
- * @throws InterruptedException exception
- */
- protected static void copyTableRegionInfo(BackupContext backupContext, Configuration conf)
- throws IOException, InterruptedException {
-
- Path rootDir = FSUtils.getRootDir(conf);
- FileSystem fs = rootDir.getFileSystem(conf);
-
- // for each table in the table set, copy out the table info and region info files in the correct
- // directory structure
- for (String table : backupContext.getTables()) {
-
- LOG.debug("Attempting to copy table info for:" + table);
- TableDescriptor orig =
- FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, TableName.valueOf(table));
-
- // write a copy of descriptor to the target directory
- Path target = new Path(backupContext.getBackupStatus(table).getTargetDir());
- FileSystem targetFs = target.getFileSystem(conf);
- FSTableDescriptors descriptors =
- new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf));
- descriptors.createTableDescriptorForTableDirectory(target, orig, false);
- LOG.debug("Finished copying tableinfo.");
-
- HBaseAdmin hbadmin = null;
- // TODO: optimize
- Connection conn = null;
- List<HRegionInfo> regions = null;
- try {
- conn = ConnectionFactory.createConnection(conf);
- hbadmin = (HBaseAdmin) conn.getAdmin();
- regions = hbadmin.getTableRegions(TableName.valueOf(table));
- } catch (Exception e) {
- throw new BackupException(e);
- } finally {
- if (hbadmin != null) {
- hbadmin.close();
- }
- if(conn != null){
- conn.close();
- }
- }
-
- // For each region, write the region info to disk
- LOG.debug("Starting to write region info for table " + table);
- for (HRegionInfo regionInfo : regions) {
- Path regionDir =
- HRegion.getRegionDir(new Path(backupContext.getBackupStatus(table).getTargetDir()),
- regionInfo);
- regionDir =
- new Path(backupContext.getBackupStatus(table).getTargetDir(), regionDir.getName());
- writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo);
- }
- LOG.debug("Finished writing region info for table " + table);
- }
- }
-
- /**
- * Write the .regioninfo file on-disk.
- */
- public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs,
- final Path regionInfoDir, HRegionInfo regionInfo) throws IOException {
- final byte[] content = regionInfo.toDelimitedByteArray();
- Path regionInfoFile = new Path(regionInfoDir, ".regioninfo");
- // First check to get the permissions
- FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
- // Write the RegionInfo file content
- FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
- try {
- out.write(content);
- } finally {
- out.close();
- }
- }
-
- /**
- * TODO: verify the code
- * @param p path
- * @return host name
- * @throws IOException exception
- */
- protected static String parseHostFromOldLog(Path p) throws IOException {
- String n = p.getName();
- int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
- String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
- return ServerName.parseHostname(s);
- }
-
- public static String parseHostNameFromLogFile(Path p) throws IOException {
- if (isArchivedLogFile(p)) {
- return parseHostFromOldLog(p);
- } else {
- return DefaultWALProvider.getServerNameFromWALDirectoryName(p).getHostname();
- }
- }
-
- private static boolean isArchivedLogFile(Path p) {
- String oldLog = Path.SEPARATOR + HConstants.HREGION_OLDLOGDIR_NAME + Path.SEPARATOR;
- return p.toString().contains(oldLog);
- }
-
- /**
- * Return WAL file name
- * @param walFileName WAL file name
- * @return WAL file name
- * @throws IOException exception
- * @throws IllegalArgumentException exception
- */
- public static String getUniqueWALFileNamePart(String walFileName) throws IOException {
- return new Path(walFileName).getName();
- }
-
- /**
- * Return WAL file name
- * @param p - WAL file path
- * @return WAL file name
- * @throws IOException exception
- */
- public static String getUniqueWALFileNamePart(Path p) throws IOException {
- return p.getName();
- }
-
- /**
- * Given the log file, parse the timestamp from the file name. The timestamp is the last number.
- * @param p a path to the log file
- * @return the timestamp
- * @throws IOException exception
- */
- protected static String getCreationTime(Path p, Configuration conf) throws IOException {
- int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);
- if (idx < 0) {
- throw new IOException("Cannot parse timestamp from path " + p);
- }
- String ts = p.getName().substring(idx + 1);
- return ts;
- }
-
- /**
- * Get the total length of files under the given directory recursively.
- * @param fs The hadoop file system
- * @param dir The target directory
- * @return the total length of files
- * @throws IOException exception
- */
- public static long getFilesLength(FileSystem fs, Path dir) throws IOException {
- long totalLength = 0;
- FileStatus[] files = FSUtils.listStatus(fs, dir);
- if (files != null) {
- for (FileStatus fileStatus : files) {
- if (fileStatus.isDir()) {
- totalLength += getFilesLength(fs, fileStatus.getPath());
- } else {
- totalLength += fileStatus.getLen();
- }
- }
- }
- return totalLength;
- }
-
- /**
- * Keep the record for dependency for incremental backup and history info p.s, we may be able to
- * merge this class into backupImage class later
- */
- public static class BackupCompleteData implements Comparable<BackupCompleteData> {
- private String startTime;
- private String endTime;
- private String type;
- private String backupRootPath;
- private String tableList;
- private String backupToken;
- private String bytesCopied;
- private List<String> ancestors;
- private boolean fromExistingSnapshot = false;
-
- public List<String> getAncestors() {
- if (fromExistingSnapshot) {
- return null;
- }
- if (this.ancestors == null) {
- this.ancestors = new ArrayList<String>();
- }
- return this.ancestors;
- }
-
- public void addAncestor(String backupToken) {
- this.getAncestors().add(backupToken);
- }
-
- public String getBytesCopied() {
- return bytesCopied;
- }
-
- public void setBytesCopied(String bytesCopied) {
- this.bytesCopied = bytesCopied;
- }
-
- public String getBackupToken() {
- return backupToken;
- }
-
- public void setBackupToken(String backupToken) {
- this.backupToken = backupToken;
- }
-
- public String getStartTime() {
- return startTime;
- }
-
- public void setStartTime(String startTime) {
- this.startTime = startTime;
- }
-
- public String getEndTime() {
- return endTime;
- }
-
- public void setEndTime(String endTime) {
- this.endTime = endTime;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getBackupRootPath() {
- return backupRootPath;
- }
-
- public void setBackupRootPath(String backupRootPath) {
- this.backupRootPath = backupRootPath;
- }
-
- public String getTableList() {
- return tableList;
- }
-
- public void setTableList(String tableList) {
- this.tableList = tableList;
- }
-
- public boolean fromExistingSnapshot() {
- return this.fromExistingSnapshot;
- }
-
- public void markFromExistingSnapshot() {
- this.fromExistingSnapshot = true;
- }
-
- @Override
- public int compareTo(BackupCompleteData o) {
- Long thisTS =
- new Long(this.getBackupToken().substring(this.getBackupToken().lastIndexOf("_") + 1));
- Long otherTS =
- new Long(o.getBackupToken().substring(o.getBackupToken().lastIndexOf("_") + 1));
- return thisTS.compareTo(otherTS);
- }
-
- }
-
- /**
- * Sort history list by start time in descending order.
- * @param historyList history list
- * @return sorted list of BackupCompleteData
- */
- public static ArrayList<BackupCompleteData> sortHistoryListDesc(
- ArrayList<BackupCompleteData> historyList) {
- ArrayList<BackupCompleteData> list = new ArrayList<BackupCompleteData>();
- TreeMap<String, BackupCompleteData> map = new TreeMap<String, BackupCompleteData>();
- for (BackupCompleteData h : historyList) {
- map.put(h.getStartTime(), h);
- }
- Iterator<String> i = map.descendingKeySet().iterator();
- while (i.hasNext()) {
- list.add(map.get(i.next()));
- }
- return list;
- }
-
- /**
- * Get list of all WAL files (WALs and archive)
- * @param c - configuration
- * @return list of WAL files
- * @throws IOException exception
- */
- public static List<String> getListOfWALFiles(Configuration c) throws IOException {
- Path rootDir = FSUtils.getRootDir(c);
- Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
- Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
- List<String> logFiles = new ArrayList<String>();
-
- FileSystem fs = FileSystem.get(c);
- logFiles = getFiles(fs, logDir, logFiles, null);
- logFiles = getFiles(fs, oldLogDir, logFiles, null);
- return logFiles;
- }
-
- /**
- * Get list of all WAL files (WALs and archive)
- * @param c - configuration
- * @return list of WAL files
- * @throws IOException exception
- */
- public static List<String> getListOfWALFiles(Configuration c, PathFilter filter)
- throws IOException {
- Path rootDir = FSUtils.getRootDir(c);
- Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
- Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
- List<String> logFiles = new ArrayList<String>();
-
- FileSystem fs = FileSystem.get(c);
- logFiles = getFiles(fs, logDir, logFiles, filter);
- logFiles = getFiles(fs, oldLogDir, logFiles, filter);
- return logFiles;
- }
-
- /**
- * Get list of all old WAL files (WALs and archive)
- * @param c - configuration
- * @return list of WAL files
- * @throws IOException exception
- */
- public static List<String> getWALFilesOlderThan(final Configuration c,
- final HashMap<String, String> hostTimestampMap) throws IOException {
- Path rootDir = FSUtils.getRootDir(c);
- Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
- Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
- List<String> logFiles = new ArrayList<String>();
-
- PathFilter filter = new PathFilter() {
-
- @Override
- public boolean accept(Path p) {
- try {
- if (DefaultWALProvider.isMetaFile(p)) {
- return false;
- }
- String host = BackupUtil.parseHostNameFromLogFile(p);
- String oldTimestamp = hostTimestampMap.get(host);
- String currentLogTS = getCreationTime(p, c);
- if (LOG.isDebugEnabled()) {
- LOG.debug("path=" + p);
- LOG.debug("oldTimestamp=" + oldTimestamp);
- LOG.debug("currentLogTS=" + currentLogTS);
- }
- return Long.parseLong(currentLogTS) <= Long.parseLong(oldTimestamp);
- } catch (IOException e) {
- LOG.error(e);
- return false;
- }
- }
- };
- FileSystem fs = FileSystem.get(c);
- logFiles = getFiles(fs, logDir, logFiles, filter);
- logFiles = getFiles(fs, oldLogDir, logFiles, filter);
- return logFiles;
- }
-
- private static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files,
- PathFilter filter) throws FileNotFoundException, IOException {
-
- RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true);
-
- while (it.hasNext()) {
- LocatedFileStatus lfs = it.next();
- if (lfs.isDirectory()) {
- continue;
- }
- // apply filter
- if (filter.accept(lfs.getPath())) {
- files.add(lfs.getPath().toString());
- LOG.info(lfs.getPath());
- }
- }
- return files;
- }
-
- public static String concat(Collection<String> col, String separator) {
- if (col.size() == 0) {
- return "";
- }
- StringBuilder sb = new StringBuilder();
- for (String s : col) {
- sb.append(s + separator);
- }
- sb.deleteCharAt(sb.lastIndexOf(";"));
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java
deleted file mode 100644
index 74411da..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HBackupFileSystem.java
+++ /dev/null
@@ -1,511 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * View to an on-disk Backup Image FileSytem
- * Provides the set of methods necessary to interact with the on-disk Backup Image data.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class HBackupFileSystem {
- public static final Log LOG = LogFactory.getLog(HBackupFileSystem.class);
-
- private final String RESTORE_TMP_PATH = "/tmp/restoreTemp";
- private final String[] ignoreDirs = { "recovered.edits" };
-
- private final Configuration conf;
- private final FileSystem fs;
- private final Path backupRootPath;
- private final String backupId;
-
- /**
- * Create a view to the on-disk Backup Image.
- * @param conf to use
- * @param backupPath to where the backup Image stored
- * @param backupId represent backup Image
- */
- HBackupFileSystem(final Configuration conf, final Path backupRootPath, final String backupId)
- throws IOException {
- this.conf = conf;
- this.fs = backupRootPath.getFileSystem(conf);
- this.backupRootPath = backupRootPath;
- this.backupId = backupId; // the backup ID for the lead backup Image
- }
-
- /**
- * @param tableName is the table backuped
- * @return {@link HTableDescriptor} saved in backup image of the table
- */
- protected HTableDescriptor getTableDesc(String tableName) throws FileNotFoundException,
- IOException {
-
- Path tableInfoPath = this.getTableInfoPath(tableName);
- LOG.debug("tableInfoPath = " + tableInfoPath.toString());
- SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
- LOG.debug("desc = " + desc.getName());
- SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
- HTableDescriptor tableDescriptor = manifest.getTableDescriptor();
- /*
- * for HBase 0.96 or 0.98 HTableDescriptor tableDescriptor =
- * FSTableDescriptors.getTableDescriptorFromFs(fs, tableInfoPath);
- */
- if (!tableDescriptor.getNameAsString().equals(tableName)) {
- LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
- + tableInfoPath.toString());
- LOG.error("tableDescriptor.getNameAsString() = " + tableDescriptor.getNameAsString());
- }
- return tableDescriptor;
- }
-
- /**
- * Given the backup root dir, backup id and the table name, return the backup image location,
- * which is also where the backup manifest file is. return value look like:
- * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/default/t1_dn/backup_1396650096738"
- * @param backupRootDir backup root directory
- * @param backupId backup id
- * @param table table name
- * @return backupPath String for the particular table
- */
- protected static String getTableBackupDir(String backupRootDir, String backupId, String table) {
- TableName tableName = TableName.valueOf(table);
- return backupRootDir + File.separator + tableName.getNamespaceAsString() + File.separator
- + tableName.getQualifierAsString() + File.separator + backupId;
- }
-
- /**
- * Given the backup root dir, backup id and the table name, return the backup image location,
- * which is also where the backup manifest file is. return value look like:
- * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/default/t1_dn/backup_1396650096738"
- * @param tableN table name
- * @return backupPath for the particular table
- */
- protected Path getTableBackupPath(String tableN) {
- TableName tableName = TableName.valueOf(tableN);
- return new Path(this.backupRootPath, tableName.getNamespaceAsString() + File.separator
- + tableName.getQualifierAsString() + File.separator + backupId);
- }
-
- /**
- * return value represent path for:
- * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/.hbase-snapshot"
- * @param tableName table name
- * @return path for snapshot
- */
- protected Path getTableSnapshotPath(String tableName) {
- return new Path(this.getTableBackupPath(tableName), HConstants.SNAPSHOT_DIR_NAME);
- }
-
- /**
- * return value represent path for:
- * "..../default/t1_dn/backup_1396650096738/.hbase-snapshot/snapshot_1396650097621_default_t1_dn"
- * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo,
- * .data.manifest (trunk)
- * @param tableName table name
- * @return path to table info
- * @throws FileNotFoundException exception
- * @throws IOException exception
- */
- protected Path getTableInfoPath(String tableName) throws FileNotFoundException, IOException {
-
- Path tableSnapShotPath = this.getTableSnapshotPath(tableName);
- Path tableInfoPath = null;
-
- // can't build the path directly as the timestamp values are different
- FileStatus[] snapshots = fs.listStatus(tableSnapShotPath);
- for (FileStatus snapshot : snapshots) {
- tableInfoPath = snapshot.getPath();
- // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest";
- if (tableInfoPath.getName().endsWith("data.manifest")) {
- LOG.debug("find Snapshot Manifest");
- break;
- }
- }
- return tableInfoPath;
- }
-
- /**
- * return value represent path for:
- * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn"
- * @param tabelName table name
- * @return path to table archive
- * @throws IOException exception
- */
- protected Path getTableArchivePath(String tableName) throws IOException {
- Path baseDir = new Path(getTableBackupPath(tableName), HConstants.HFILE_ARCHIVE_DIRECTORY);
- Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR);
- Path archivePath = new Path(dataDir, TableName.valueOf(tableName).getNamespaceAsString());
- Path tableArchivePath =
- new Path(archivePath, TableName.valueOf(tableName).getQualifierAsString());
- if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) {
- LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists");
- tableArchivePath = null; // empty table has no archive
- }
- return tableArchivePath;
- }
-
- /**
- * Given the backup root dir and the backup id, return the log file location for an incremental
- * backup.
- * @param backupRootDir backup root directory
- * @param backupId backup id
- * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738"
- */
- protected static String getLogBackupDir(String backupRootDir, String backupId) {
- return backupRootDir + File.separator + HConstants.HREGION_LOGDIR_NAME + File.separator
- + backupId;
- }
-
- protected static Path getLogBackupPath(String backupRootDir, String backupId) {
- return new Path(getLogBackupDir(backupRootDir, backupId));
- }
-
- private Path getManifestPath(String tableName) throws IOException {
- Path manifestPath = new Path(getTableBackupPath(tableName), BackupManifest.FILE_NAME);
-
- LOG.debug("Looking for " + manifestPath.toString());
- if (!fs.exists(manifestPath)) {
- // check log dir for incremental backup case
- manifestPath =
- new Path(getLogBackupDir(this.backupRootPath.toString(), this.backupId) + File.separator
- + BackupManifest.FILE_NAME);
- LOG.debug("Looking for " + manifestPath.toString());
- if (!fs.exists(manifestPath)) {
- String errorMsg =
- "Could not find backup manifest for " + backupId + " in " + backupRootPath.toString();
- throw new IOException(errorMsg);
- }
- }
- return manifestPath;
- }
-
- protected BackupManifest getManifest(String tableName) throws IOException {
- BackupManifest manifest = new BackupManifest(conf, this.getManifestPath(tableName));
- return manifest;
- }
-
- /**
- * Gets region list
- * @param tableName table name
- * @return RegionList region list
- * @throws FileNotFoundException exception
- * @throws IOException exception
- */
-
- protected ArrayList<Path> getRegionList(String tableName) throws FileNotFoundException,
- IOException {
- Path tableArchivePath = this.getTableArchivePath(tableName);
- ArrayList<Path> regionDirList = new ArrayList<Path>();
- FileStatus[] children = fs.listStatus(tableArchivePath);
- for (FileStatus childStatus : children) {
- // here child refer to each region(Name)
- Path child = childStatus.getPath();
- regionDirList.add(child);
- }
- return regionDirList;
- }
-
- /**
- * Gets region list
- * @param tableArchivePath table archive path
- * @return RegionList region list
- * @throws FileNotFoundException exception
- * @throws IOException exception
- */
- protected ArrayList<Path> getRegionList(Path tableArchivePath) throws FileNotFoundException,
- IOException {
- ArrayList<Path> regionDirList = new ArrayList<Path>();
- FileStatus[] children = fs.listStatus(tableArchivePath);
- for (FileStatus childStatus : children) {
- // here child refer to each region(Name)
- Path child = childStatus.getPath();
- regionDirList.add(child);
- }
- return regionDirList;
- }
-
- /**
- * Counts the number of files in all subdirectories of an HBase tables, i.e. HFiles. And finds the
- * maximum number of files in one HBase table.
- * @param tableArchivePath archive path
- * @return the maximum number of files found in 1 HBase table
- * @throws IOException exception
- */
- protected int getMaxNumberOfFilesInSubDir(Path tableArchivePath) throws IOException {
- int result = 1;
- ArrayList<Path> regionPathList = this.getRegionList(tableArchivePath);
- // tableArchivePath = this.getTableArchivePath(tableName);
-
- if (regionPathList == null || regionPathList.size() == 0) {
- throw new IllegalStateException("Cannot restore hbase table because directory '"
- + tableArchivePath + "' is not a directory.");
- }
-
- for (Path regionPath : regionPathList) {
- result = Math.max(result, getNumberOfFilesInDir(regionPath));
- }
- return result;
- }
-
- /**
- * Counts the number of files in all subdirectories of an HBase table, i.e. HFiles.
- * @param regionPath Path to an HBase table directory
- * @return the number of files all directories
- * @throws IOException exception
- */
- protected int getNumberOfFilesInDir(Path regionPath) throws IOException {
- int result = 0;
-
- if (!fs.exists(regionPath) || !fs.getFileStatus(regionPath).isDirectory()) {
- throw new IllegalStateException("Cannot restore hbase table because directory '"
- + regionPath.toString() + "' is not a directory.");
- }
-
- FileStatus[] tableDirContent = fs.listStatus(regionPath);
- for (FileStatus subDirStatus : tableDirContent) {
- FileStatus[] colFamilies = fs.listStatus(subDirStatus.getPath());
- for (FileStatus colFamilyStatus : colFamilies) {
- FileStatus[] colFamilyContent = fs.listStatus(colFamilyStatus.getPath());
- result += colFamilyContent.length;
- }
- }
- return result;
- }
-
- /**
- * Duplicate the backup image if it's on local cluster
- * @see HStore#bulkLoadHFile(String, long)
- * @see HRegionFileSystem#bulkLoadStoreFile(String familyName, Path srcPath, long seqNum)
- * @param tableArchivePath archive path
- * @return the new tableArchivePath
- * @throws IOException exception
- */
- protected Path checkLocalAndBackup(Path tableArchivePath) throws IOException {
- // Move the file if it's on local cluster
- boolean isCopyNeeded = false;
-
- FileSystem srcFs = tableArchivePath.getFileSystem(conf);
- FileSystem desFs = FileSystem.get(conf);
- if (tableArchivePath.getName().startsWith("/")) {
- isCopyNeeded = true;
- } else {
- // This should match what is done in @see HRegionFileSystem#bulkLoadStoreFile(String, Path,
- // long)
- if (srcFs.getUri().equals(desFs.getUri())) {
- LOG.debug("cluster hold the backup image: " + srcFs.getUri() + "; local cluster node: "
- + desFs.getUri());
- isCopyNeeded = true;
- }
- }
- if (isCopyNeeded) {
- LOG.debug("File " + tableArchivePath + " on local cluster, back it up before restore");
- Path tmpPath = new Path(RESTORE_TMP_PATH);
- if (desFs.exists(tmpPath)) {
- try {
- desFs.delete(tmpPath, true);
- } catch (IOException e) {
- LOG.debug("Failed to delete path: " + tmpPath
- + ", need to check whether restore target DFS cluster is healthy");
- }
- }
- FileUtil.copy(srcFs, tableArchivePath, desFs, tmpPath, false, conf);
- LOG.debug("Copied to temporary path on local cluster: " + tmpPath);
- tableArchivePath = tmpPath;
- }
- return tableArchivePath;
- }
-
- /**
- * Calculate region boundaries and add all the column families to the table descriptor
- * @param regionDirList region dir list
- * @return a set of keys to store the boundaries
- */
- protected byte[][] generateBoundaryKeys(ArrayList<Path> regionDirList)
- throws FileNotFoundException, IOException {
- TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
- // Build a set of keys to store the boundaries
- byte[][] keys = null;
- // calculate region boundaries and add all the column families to the table descriptor
- for (Path regionDir : regionDirList) {
- LOG.debug("Parsing region dir: " + regionDir);
- Path hfofDir = regionDir;
-
- if (!fs.exists(hfofDir)) {
- LOG.warn("HFileOutputFormat dir " + hfofDir + " not found");
- }
-
- FileStatus[] familyDirStatuses = fs.listStatus(hfofDir);
- if (familyDirStatuses == null) {
- throw new IOException("No families found in " + hfofDir);
- }
-
- for (FileStatus stat : familyDirStatuses) {
- if (!stat.isDirectory()) {
- LOG.warn("Skipping non-directory " + stat.getPath());
- continue;
- }
- boolean isIgnore = false;
- String pathName = stat.getPath().getName();
- for (String ignore : ignoreDirs) {
- if (pathName.contains(ignore)) {
- LOG.warn("Skipping non-family directory" + pathName);
- isIgnore = true;
- break;
- }
- }
- if (isIgnore) {
- continue;
- }
- Path familyDir = stat.getPath();
- LOG.debug("Parsing family dir [" + familyDir.toString() + " in region [" + regionDir + "]");
- // Skip _logs, etc
- if (familyDir.getName().startsWith("_") || familyDir.getName().startsWith(".")) {
- continue;
- }
-
- // start to parse hfile inside one family dir
- Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
- for (Path hfile : hfiles) {
- if (hfile.getName().startsWith("_") || hfile.getName().startsWith(".")
- || StoreFileInfo.isReference(hfile.getName())
- || HFileLink.isHFileLink(hfile.getName())) {
- continue;
- }
- HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(conf), conf);
- final byte[] first, last;
- try {
- reader.loadFileInfo();
- first = reader.getFirstRowKey();
- last = reader.getLastRowKey();
- LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first="
- + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last));
-
- // To eventually infer start key-end key boundaries
- Integer value = map.containsKey(first) ? (Integer) map.get(first) : 0;
- map.put(first, value + 1);
- value = map.containsKey(last) ? (Integer) map.get(last) : 0;
- map.put(last, value - 1);
- } finally {
- reader.close();
- }
- }
- }
- }
- keys = LoadIncrementalHFiles.inferBoundaries(map);
- return keys;
- }
-
- /**
- * Check whether the backup path exist
- * @param backupStr backup
- * @param conf configuration
- * @return Yes if path exists
- * @throws IOException exception
- */
- protected static boolean checkPathExist(String backupStr, Configuration conf)
- throws IOException {
- boolean isExist = false;
- Path backupPath = new Path(backupStr);
- FileSystem fileSys = backupPath.getFileSystem(conf);
- String targetFsScheme = fileSys.getUri().getScheme();
- LOG.debug("Schema of given url: " + backupStr + " is: " + targetFsScheme);
- if (fileSys.exists(backupPath)) {
- isExist = true;
- }
- return isExist;
- }
-
- /**
- * Check whether the backup image path and there is manifest file in the path.
- * @param backupManifestMap If all the manifests are found, then they are put into this map
- * @param tableArray the tables involved
- * @throws IOException exception
- */
- protected void checkImageManifestExist(HashMap<String, BackupManifest> backupManifestMap,
- String[] tableArray) throws IOException {
-
- try {
- for (String tableName : tableArray) {
- BackupManifest manifest = this.getManifest(tableName);
- backupManifestMap.put(tableName, manifest);
- }
- } catch (IOException e) {
- String expMsg = e.getMessage();
- if (expMsg.contains("No FileSystem for scheme")) {
- if (expMsg.contains("gpfs")) {
- LOG.error("Please change to use webhdfs url when "
- + "the backup image to restore locates on gpfs cluster");
- } else {
- LOG.error("Unsupported filesystem scheme found in the backup target url, "
- + "please check the url to make sure no typo in it");
- }
- throw e;
- } else if (expMsg.contains("no authority supported")) {
- LOG.error("Please change to use webhdfs url when "
- + "the backup image to restore locates on gpfs cluster");
- throw e;
- } else {
- LOG.error(expMsg);
- throw e;
- }
- }
- }
-
- public static String join(String[] names) {
- StringBuilder sb = new StringBuilder();
- String sep = BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND;
- for (String s : names) {
- sb.append(sep).append(s);
- }
- return sb.toString();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java
deleted file mode 100644
index e91857f..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalBackupManager.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.wal.DefaultWALProvider;
-
-/**
- * After a full backup was created, the incremental backup will only store the changes made
- * after the last full or incremental backup.
- *
- * Creating the backup copies the logfiles in .logs and .oldlogs since the last backup timestamp.
- *
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class IncrementalBackupManager {
- // parent manager
- private BackupManager backupManager;
-
- public static final Log LOG = LogFactory.getLog(IncrementalBackupManager.class);
-
- public IncrementalBackupManager(BackupManager bm) {
- this.backupManager = bm;
- }
-
- /**
- * Obtain the list of logs that need to be copied out for this incremental backup. The list is set
- * in BackupContext.
- * @param backupContext backup context
- * @return The new HashMap of RS log timestamps after the log roll for this incremental backup.
- * @throws IOException exception
- */
- public HashMap<String, String> getIncrBackupLogFileList(BackupContext backupContext)
- throws IOException {
- List<String> logList;
- HashMap<String, String> newTimestamps;
- HashMap<String, String> previousTimestampMins;
-
- Configuration conf = BackupUtil.getConf();
- String savedStartCode = backupManager.readBackupStartCode();
-
- // key: tableName
- // value: <RegionServer,PreviousTimeStamp>
- HashMap<String, HashMap<String, String>> previousTimestampMap =
- backupManager.readLogTimestampMap();
-
- previousTimestampMins = BackupUtil.getRSLogTimestampMins(previousTimestampMap);
-
- LOG.debug("StartCode " + savedStartCode + "for backupID " + backupContext.getBackupId());
- LOG.debug("Timestamps " + previousTimestampMap);
- // get all new log files from .logs and .oldlogs after last TS and before new timestamp
- if (savedStartCode == null ||
- previousTimestampMins == null ||
- previousTimestampMins.isEmpty()) {
- throw new IOException("Cannot read any previous back up timestamps from hbase:backup. "
- + "In order to create an incremental backup, at least one full backup is needed.");
- }
-
- HBaseAdmin hbadmin = null;
- Connection conn = null;
- try {
- LOG.info("Execute roll log procedure for incremental backup ...");
- conn = ConnectionFactory.createConnection(conf);
- hbadmin = (HBaseAdmin) conn.getAdmin();
- hbadmin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
- LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, new HashMap<String, String>());
- } finally {
- if (hbadmin != null) {
- hbadmin.close();
- }
- if(conn != null){
- conn.close();
- }
- }
-
- newTimestamps = backupManager.readRegionServerLastLogRollResult();
-
- logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps, conf, savedStartCode);
-
- backupContext.setIncrBackupFileList(logList);
-
- return newTimestamps;
- }
-
- /**
- * For each region server: get all log files newer than the last timestamps but not newer than the
- * newest timestamps.
- * @param olderTimestamps the timestamp for each region server of the last backup.
- * @param newestTimestamps the timestamp for each region server that the backup should lead to.
- * @param conf the Hadoop and Hbase configuration
- * @param savedStartCode the startcode (timestamp) of last successful backup.
- * @return a list of log files to be backed up
- * @throws IOException exception
- */
- private List<String> getLogFilesForNewBackup(HashMap<String, String> olderTimestamps,
- HashMap<String, String> newestTimestamps, Configuration conf, String savedStartCode)
- throws IOException {
- LOG.debug("In getLogFilesForNewBackup()\n" + "olderTimestamps: " + olderTimestamps
- + "\n newestTimestamps: " + newestTimestamps);
- Path rootdir = FSUtils.getRootDir(conf);
- Path logDir = new Path(rootdir, HConstants.HREGION_LOGDIR_NAME);
- Path oldLogDir = new Path(rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
- FileSystem fs = rootdir.getFileSystem(conf);
- NewestLogFilter pathFilter = new NewestLogFilter(conf);
-
- List<String> resultLogFiles = new ArrayList<String>();
- List<String> newestLogs = new ArrayList<String>();
-
- /*
- * The old region servers and timestamps info we kept in hbase:backup may be out of sync if new
- * region server is added or existing one lost. We'll deal with it here when processing the
- * logs. If data in hbase:backup has more hosts, just ignore it. If the .logs directory includes
- * more hosts, the additional hosts will not have old timestamps to compare with. We'll just use
- * all the logs in that directory. We always write up-to-date region server and timestamp info
- * to hbase:backup at the end of successful backup.
- */
-
- FileStatus[] rss;
- Path p;
- String host;
- String oldTimeStamp;
- String currentLogFile;
- String currentLogTS;
-
- // Get the files in .logs.
- rss = fs.listStatus(logDir);
- for (FileStatus rs : rss) {
- p = rs.getPath();
- host = DefaultWALProvider.getServerNameFromWALDirectoryName(p).getHostname();
- FileStatus[] logs;
- oldTimeStamp = olderTimestamps.get(host);
- // It is possible that there is no old timestamp in hbase:backup for this host if
- // this region server is newly added after our last backup.
- if (oldTimeStamp == null) {
- logs = fs.listStatus(p);
- } else {
- pathFilter.setLastBackupTS(oldTimeStamp);
- logs = fs.listStatus(p, pathFilter);
- }
- for (FileStatus log : logs) {
- LOG.debug("currentLogFile: " + log.getPath().toString());
- if (DefaultWALProvider.isMetaFile(log.getPath())) {
- LOG.debug("Skip hbase:meta log file: " + log.getPath().getName());
- continue;
- }
- currentLogFile = log.getPath().toString();
- resultLogFiles.add(currentLogFile);
- currentLogTS = BackupUtil.getCreationTime(log.getPath(), conf);
- // newestTimestamps is up-to-date with the current list of hosts
- // so newestTimestamps.get(host) will not be null.
- if (Long.valueOf(currentLogTS) > Long.valueOf(newestTimestamps.get(host))) {
- newestLogs.add(currentLogFile);
- }
- }
- }
-
- // Include the .oldlogs files too.
- FileStatus[] oldlogs = fs.listStatus(oldLogDir);
- for (FileStatus oldlog : oldlogs) {
- p = oldlog.getPath();
- currentLogFile = p.toString();
- if (DefaultWALProvider.isMetaFile(p)) {
- LOG.debug("Skip .meta log file: " + currentLogFile);
- continue;
- }
- host = BackupUtil.parseHostFromOldLog(p);
- currentLogTS = BackupUtil.getCreationTime(p, conf);
- oldTimeStamp = olderTimestamps.get(host);
- /*
- * It is possible that there is no old timestamp in hbase:backup for this host. At the time of
- * our last backup operation, this rs did not exist. The reason can be one of the two: 1. The
- * rs already left/crashed. Its logs were moved to .oldlogs. 2. The rs was added after our
- * last backup.
- */
- if (oldTimeStamp == null) {
- if (Long.valueOf(currentLogTS) < Long.valueOf(savedStartCode)) {
- // This log file is really old, its region server was before our last backup.
- continue;
- } else {
- resultLogFiles.add(currentLogFile);
- }
- } else if (Long.valueOf(currentLogTS) > Long.valueOf(oldTimeStamp)) {
- resultLogFiles.add(currentLogFile);
- }
-
- LOG.debug("resultLogFiles before removal of newestLogs: " + resultLogFiles);
- // It is possible that a host in .oldlogs is an obsolete region server
- // so newestTimestamps.get(host) here can be null.
- // Even if these logs belong to a obsolete region server, we still need
- // to include they to avoid loss of edits for backup.
- String newTimestamp = newestTimestamps.get(host);
- if (newTimestamp != null && Long.valueOf(currentLogTS) > Long.valueOf(newTimestamp)) {
- newestLogs.add(currentLogFile);
- }
- }
- LOG.debug("newestLogs: " + newestLogs);
- // remove newest log per host because they are still in use
- resultLogFiles.removeAll(newestLogs);
- LOG.debug("resultLogFiles after removal of newestLogs: " + resultLogFiles);
- return resultLogFiles;
- }
-
- class NewestLogFilter implements PathFilter {
- private String lastBackupTS = "0";
- final private Configuration conf;
-
- public NewestLogFilter(Configuration conf) {
- this.conf = conf;
- }
-
- protected void setLastBackupTS(String ts) {
- this.lastBackupTS = ts;
- }
-
- @Override
- public boolean accept(Path path) {
- // skip meta table log -- ts.meta file
- if (DefaultWALProvider.isMetaFile(path)) {
- LOG.debug("Skip .meta log file: " + path.getName());
- return false;
- }
- String timestamp;
- try {
- timestamp = BackupUtil.getCreationTime(path, conf);
- return Long.valueOf(timestamp) > Long.valueOf(lastBackupTS);
- } catch (IOException e) {
- LOG.warn("Cannot read timestamp of log file " + path);
- return false;
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java
deleted file mode 100644
index 72e4879..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/IncrementalRestoreService.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface IncrementalRestoreService extends Configurable{
-
- public void run(String logDirectory, String[] fromTables, String[] toTables)
- throws IOException;
-}
[07/29] hbase git commit: HBASE-14030 Revert due to pending review
comments
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
deleted file mode 100644
index ad23f9b..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestRestoreBoundaryTests extends TestBackupBase {
-
- private static final Log LOG = LogFactory.getLog(TestRestoreBoundaryTests.class);
-
- /**
- * Verify that a single empty table is restored to a new table
- * @throws Exception
- */
- @Test
- public void testFullRestoreSingleEmpty() throws Exception {
-
- LOG.info("test full restore on a single table empty table");
- String backupId =
- BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null);
- LOG.info("backup complete");
- assertTrue(checkSucceeded(backupId));
- String[] tableset = new String[] { table1.getNameAsString() };
- String[] tablemap = new String[] { table1_restore };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap,
- false);
- HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
- assertTrue(hba.tableExists(TableName.valueOf(table1_restore)));
- TEST_UTIL.deleteTable(TableName.valueOf(table1_restore));
- }
-
- /**
- * Verify that multiple tables are restored to new tables.
- * @throws Exception
- */
- @Test
- public void testFullRestoreMultipleEmpty() throws Exception {
- LOG.info("create full backup image on multiple tables");
- String tableset =
- table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND
- + table3.getNameAsString();
- String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null);
- assertTrue(checkSucceeded(backupId));
- String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() };
- String[] tablemap = new String[] { table2_restore, table3_restore };
- Path path = new Path(BACKUP_ROOT_DIR);
- HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId);
- RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, restore_tableset,
- tablemap,
- false);
- HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
- assertTrue(hba.tableExists(TableName.valueOf(table2_restore)));
- assertTrue(hba.tableExists(TableName.valueOf(table3_restore)));
- TEST_UTIL.deleteTable(TableName.valueOf(table2_restore));
- TEST_UTIL.deleteTable(TableName.valueOf(table3_restore));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
index cd2efad..7620bbb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/SimpleRSProcedureManager.java
@@ -49,7 +49,7 @@ public class SimpleRSProcedureManager extends RegionServerProcedureManager {
private ProcedureMember member;
@Override
- public void initialize(RegionServerServices rss) throws IOException {
+ public void initialize(RegionServerServices rss) throws KeeperException {
this.rss = rss;
ZooKeeperWatcher zkw = rss.getZooKeeper();
this.memberRpcs = new ZKProcedureMemberRpcs(zkw, getProcedureSignature());
[05/29] hbase git commit: HBASE-14030 HBase Backup/Restore Phase 1
(Vladimir Rodionov)
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java
new file mode 100644
index 0000000..f764f18
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupHandler.java
@@ -0,0 +1,744 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupManifest.BackupImage;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+/**
+ * A Handler to carry the operations of backup progress
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupHandler implements Callable<Object> {
+ private static final Log LOG = LogFactory.getLog(BackupHandler.class);
+
+ // backup phase
+ // for overall backup (for table list, some table may go online, while some may go offline)
+ protected static enum BACKUPPHASE {
+ REQUEST, SNAPSHOT, PREPARE_INCREMENTAL, SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST;
+ }
+
+ // backup status flag
+ protected static enum BACKUPSTATUS {
+ WAITING, ONGOING, COMPLETE, FAILED, CANCELLED;
+ }
+
+ protected BackupContext backupContext;
+ private BackupManager backupManager;
+ private Configuration conf;
+
+ public BackupHandler(BackupContext backupContext,
+ BackupManager backupManager, Configuration conf) {
+ this.backupContext = backupContext;
+ this.backupManager = backupManager;
+ this.conf = conf;
+ }
+
+ public BackupContext getBackupContext() {
+ return backupContext;
+ }
+
+ @Override
+ public Object call() throws Exception {
+ try {
+ // overall backup begin
+ this.beginBackup(backupContext);
+ HashMap<String, String> newTimestamps = null;
+ boolean fromExistingSnapshot = false; // supported by future jira
+ // handle full or incremental backup for table or table list
+ if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+ String savedStartCode = null;
+ HBaseAdmin hbadmin = null;
+ Connection conn = null;
+ boolean firstBackup = false;
+ // do snapshot for full table backup, if backing up from existing snapshot, then skip the
+ // step of taking snapshot
+ if (fromExistingSnapshot) {
+ LOG.error("Backup from existing snapshot, so skip the snapshot step. ");
+ LOG.error("This feature will be supported by a future jira");
+ throw new RuntimeException("Backup from existing snapshot is not supported");
+ } else {
+ try {
+ savedStartCode = backupManager.readBackupStartCode();
+ firstBackup = savedStartCode == null;
+ if (firstBackup) {
+ // This is our first backup. Let's put some marker on ZK so that we can hold the logs
+ // while we do the backup.
+ backupManager.writeBackupStartCode("0");
+ }
+ // We roll log here before we do the snapshot. It is possible there is duplicate data
+ // in the log that is already in the snapshot. But if we do it after the snapshot, we
+ // could have data loss.
+ // A better approach is to do the roll log on each RS in the same global procedure as
+ // the snapshot.
+ LOG.info("Execute roll log procedure for full backup ...");
+ conn = ConnectionFactory.createConnection(conf);
+ hbadmin = (HBaseAdmin) conn.getAdmin();
+ hbadmin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
+ LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, new HashMap<String, String>());
+ newTimestamps = backupManager.readRegionServerLastLogRollResult();
+ if (firstBackup) {
+ // Updates registered log files
+ // We record ALL old WAL files as registered, because
+ // this is a first full backup in the system and these
+ // files are not needed for next incremental backup
+ List<String> logFiles = BackupUtil.getWALFilesOlderThan(conf, newTimestamps);
+ backupManager.recordWALFiles(logFiles);
+ }
+ this.snapshotForFullBackup(backupContext);
+ } catch (BackupException e) {
+ // fail the overall backup and return
+ this.failBackup(backupContext, e, "Unexpected BackupException : ");
+ return null;
+ } finally {
+ if (hbadmin != null) {
+ hbadmin.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ // update the faked progress currently for snapshot done
+ this.updateProgress("10.0%", 0);
+ // do snapshot copy
+ try {
+ this.snapshotCopy(backupContext);
+ } catch (Exception e) {
+ // fail the overall backup and return
+ this.failBackup(backupContext, e, "Unexpected BackupException : ");
+ return null;
+ }
+ // Updates incremental backup table set
+ backupManager.addIncrementalBackupTableSet(backupContext.getTables());
+
+ } else if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ LOG.info("For incremental backup, current table set is "
+ + backupManager.getIncrementalBackupTableSet());
+ // do incremental table backup preparation
+ backupContext.setPhase(BACKUPPHASE.PREPARE_INCREMENTAL);
+ // avoid action if has been cancelled
+ if (backupContext.isCancelled()) {
+ return null;
+ }
+ try {
+ IncrementalBackupManager incrBackupManager = new IncrementalBackupManager(backupManager);
+
+ newTimestamps = incrBackupManager.getIncrBackupLogFileList(backupContext);
+ } catch (Exception e) {
+ // fail the overall backup and return
+ this.failBackup(backupContext, e, "Unexpected Exception : ");
+ return null;
+ }
+ // update the faked progress currently for incremental preparation done
+ this.updateProgress("10.0%", 0);
+
+ // do incremental copy
+ try {
+ // copy out the table and region info files for each table
+ BackupUtil.copyTableRegionInfo(backupContext, conf);
+ this.incrementalCopy(backupContext);
+ // Save list of WAL files copied
+ backupManager.recordWALFiles(backupContext.getIncrBackupFileList());
+ } catch (Exception e) {
+ // fail the overall backup and return
+ this.failBackup(backupContext, e, "Unexpected exception doing incremental copy : ");
+ return null;
+ }
+
+ } else {
+ LOG.error("Unsupport backup type: " + backupContext.getType());
+ }
+
+ // set overall backup status: complete. Here we make sure to complete the backup. After this
+ // checkpoint, even if entering cancel process, will let the backup finished
+ backupContext.setFlag(BACKUPSTATUS.COMPLETE);
+
+ if (!fromExistingSnapshot) {
+ if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ // Set the previousTimestampMap which is before this current log roll to the manifest.
+ HashMap<String, HashMap<String, String>> previousTimestampMap =
+ backupManager.readLogTimestampMap();
+ backupContext.setIncrTimestampMap(previousTimestampMap);
+ }
+ // The table list in backupContext is good for both full backup and incremental backup.
+ // For incremental backup, it contains the incremental backup table set.
+
+ backupManager.writeRegionServerLogTimestamp(backupContext.getTables(), newTimestamps);
+
+ HashMap<String, HashMap<String, String>> newTableSetTimestampMap =
+ backupManager.readLogTimestampMap();
+
+ String newStartCode =
+ BackupUtil.getMinValue(BackupUtil.getRSLogTimestampMins(newTableSetTimestampMap));
+ backupManager.writeBackupStartCode(newStartCode);
+ }
+
+ // backup complete
+ this.completeBackup(backupContext);
+ } catch (Exception e) {
+ // even during completing backup (#completeBackup(backupContext)), exception may occur, or
+ // exception occur during other process, fail the backup finally
+ this.failBackup(backupContext, e, "Error caught during backup progress: ");
+ }
+ return null;
+ }
+
+ /**
+ * Begin the overall backup.
+ * @param backupContext backup context
+ * @throws IOException exception
+ */
+ private void beginBackup(BackupContext backupContext) throws IOException {
+
+ // set the start timestamp of the overall backup
+ long startTs = EnvironmentEdgeManager.currentTime();
+ backupContext.setStartTs(startTs);
+ // set overall backup status: ongoing
+ backupContext.setFlag(BACKUPSTATUS.ONGOING);
+ LOG.info("Backup " + backupContext.getBackupId() + " starts at " + startTs + ".");
+
+ backupManager.updateBackupStatus(backupContext);
+ LOG.debug("Backup session " + backupContext.getBackupId() + " has been started.");
+
+ }
+
+ /**
+ * Snapshot for full table backup.
+ * @param backupContext backup context
+ * @throws IOException exception
+ */
+ private void snapshotForFullBackup(BackupContext backupContext) throws IOException {
+
+ LOG.info("HBase snapshot full backup for " + backupContext.getBackupId());
+
+ // avoid action if has been cancelled
+ if (backupContext.isCancelled()) {
+ return;
+ }
+
+ HBaseAdmin hbadmin = null;
+ Connection conn = null;
+
+ // we do HBase snapshot for tables in the table list one by one currently
+ for (String table : backupContext.getTables()) {
+
+ // avoid action if it has been cancelled
+ if (backupContext.isCancelled()) {
+ return;
+ }
+
+ HBaseProtos.SnapshotDescription backupSnapshot;
+ try {
+ // wrap a SnapshotDescription for offline/online snapshot
+ backupSnapshot = this.wrapSnapshotDescription(table);
+
+ // set the snapshot name in BackupStatus of this table
+ backupContext.setSnapshotName(table, backupSnapshot.getName());
+
+ // Kick off snapshot for backup
+ conn = ConnectionFactory.createConnection(conf);
+ hbadmin = (HBaseAdmin) conn.getAdmin();
+ hbadmin.snapshot(backupSnapshot);
+
+ if (LOG.isDebugEnabled() == false) {
+ // In DEBUG mode we log message already.
+ // This is not to duplicate that message.
+ LOG.info("Snapshot has been launched, waiting to finish ...");
+ }
+
+ } catch (Exception e) {
+ LOG.error("Snapshot failed to create " + getMessage(e));
+
+ // currently, we fail the overall backup if any table in the list failed, so throw the
+ // exception out for overall backup failing
+ throw new BackupException("Backup snapshot failed on table " + table, e);
+ } finally {
+ if (hbadmin != null) {
+ hbadmin.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+
+ // set the snapshot name in BackupStatus of this table, only after snapshot success.
+ backupContext.setSnapshotName(table, backupSnapshot.getName());
+
+ } // for each table in the backup table list
+
+ }
+
+ /**
+ * Fail the overall backup.
+ * @param backupContext backup context
+ * @param e exception
+ * @throws Exception exception
+ */
+ private void failBackup(BackupContext backupContext, Exception e, String msg) throws Exception {
+
+ LOG.error(msg + getMessage(e));
+
+ // If this is a cancel exception, then we've already cleaned.
+
+ if (this.backupContext.getFlag().equals(BACKUPSTATUS.CANCELLED)) {
+ return;
+ }
+
+ // set the failure timestamp of the overall backup
+ backupContext.setEndTs(EnvironmentEdgeManager.currentTime());
+
+ // set failure message
+ backupContext.setFailedMsg(e.getMessage());
+
+ // set overall backup status: failed
+ backupContext.setFlag(BACKUPSTATUS.FAILED);
+
+ // compose the backup failed data
+ String backupFailedData =
+ "BackupId=" + backupContext.getBackupId() + ",startts=" + backupContext.getStartTs()
+ + ",failedts=" + backupContext.getEndTs() + ",failedphase=" + backupContext.getPhase()
+ + ",failedmessage=" + backupContext.getFailedMsg();
+ LOG.error(backupFailedData);
+
+ backupManager.updateBackupStatus(backupContext);
+
+ // if full backup, then delete HBase snapshots if there already have snapshots taken
+ // and also clean up export snapshot log files if exist
+ if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+ if (!backupContext.fromExistingSnapshot()) {
+ this.deleteSnapshot(backupContext);
+ }
+ this.cleanupExportSnapshotLog();
+ } /*
+ * else { // support incremental backup code in future jira // TODO. See HBASE-14124 }
+ */
+
+ // clean up the uncompleted data at target directory if the ongoing backup has already entered
+ // the copy phase
+ // For incremental backup, DistCp logs will be cleaned with the targetDir.
+ this.cleanupTargetDir();
+
+ LOG.info("Backup " + backupContext.getBackupId() + " failed.");
+ }
+
+ /**
+ * Update the ongoing back token znode with new progress.
+ * @param newProgress progress
+ * @param bytesCopied bytes copied
+ * @throws NoNodeException exception
+ */
+ public void updateProgress(String newProgress, long bytesCopied) throws IOException {
+
+ // compose the new backup progress data, using fake number for now
+ String backupProgressData = newProgress;
+
+ backupContext.setProgress(newProgress);
+ backupManager.updateBackupStatus(backupContext);
+ LOG.debug("Backup progress data \"" + backupProgressData
+ + "\" has been updated to hbase:backup for " + backupContext.getBackupId());
+ }
+
+ /**
+ * Complete the overall backup.
+ * @param backupContext backup context
+ * @throws Exception exception
+ */
+ private void completeBackup(BackupContext backupContext) throws Exception {
+
+ // set the complete timestamp of the overall backup
+ backupContext.setEndTs(EnvironmentEdgeManager.currentTime());
+ // set overall backup status: complete
+ backupContext.setFlag(BACKUPSTATUS.COMPLETE);
+ // add and store the manifest for the backup
+ this.addManifest(backupContext);
+
+ // after major steps done and manifest persisted, do convert if needed for incremental backup
+ /* in-fly convert code here, provided by future jira */
+ LOG.debug("in-fly convert code here, provided by future jira");
+
+ // compose the backup complete data
+ String backupCompleteData =
+ this.obtainBackupMetaDataStr(backupContext) + ",startts=" + backupContext.getStartTs()
+ + ",completets=" + backupContext.getEndTs() + ",bytescopied="
+ + backupContext.getTotalBytesCopied();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Backup " + backupContext.getBackupId() + " finished: " + backupCompleteData);
+ }
+ backupManager.updateBackupStatus(backupContext);
+
+ // when full backup is done:
+ // - delete HBase snapshot
+ // - clean up directories with prefix "exportSnapshot-", which are generated when exporting
+ // snapshots
+ if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+ if (!backupContext.fromExistingSnapshot()) {
+ this.deleteSnapshot(backupContext);
+ }
+ this.cleanupExportSnapshotLog();
+ } else if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ this.cleanupDistCpLog();
+ } else {
+ LOG.error(" other backup types have not been implemented yet");
+ }
+
+ LOG.info("Backup " + backupContext.getBackupId() + " completed.");
+ }
+
+ /**
+ * Get backup request meta data dir as string.
+ * @param backupContext backup context
+ * @return meta data dir
+ */
+ private String obtainBackupMetaDataStr(BackupContext backupContext) {
+ StringBuffer sb = new StringBuffer();
+ sb.append("type=" + backupContext.getType() + ",tablelist=");
+ for (String table : backupContext.getTables()) {
+ sb.append(table + ";");
+ }
+ if (sb.lastIndexOf(";") > 0) {
+ sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1);
+ }
+ sb.append(",targetRootDir=" + backupContext.getTargetRootDir());
+ if (backupContext.fromExistingSnapshot()) {
+ sb.append(",snapshot=" + backupContext.getExistingSnapshot());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Do snapshot copy.
+ * @param backupContext backup context
+ * @throws Exception exception
+ */
+ private void snapshotCopy(BackupContext backupContext) throws Exception {
+
+ LOG.info("Snapshot copy is starting.");
+
+ // set overall backup phase: snapshot_copy
+ backupContext.setPhase(BACKUPPHASE.SNAPSHOTCOPY);
+
+ // avoid action if has been cancelled
+ if (backupContext.isCancelled()) {
+ return;
+ }
+
+ // call ExportSnapshot to copy files based on hbase snapshot for backup
+ // ExportSnapshot only support single snapshot export, need loop for multiple tables case
+ BackupCopyService copyService = BackupRestoreServiceFactory.getBackupCopyService(conf);
+
+ // number of snapshots matches number of tables
+ float numOfSnapshots = backupContext.getSnapshotNames().size();
+
+ LOG.debug("There are " + (int) numOfSnapshots + " snapshots to be copied.");
+
+ for (String table : backupContext.getTables()) {
+ // Currently we simply set the sub copy tasks by counting the table snapshot number, we can
+ // calculate the real files' size for the percentage in the future.
+ // TODO this below
+ // backupCopier.setSubTaskPercntgInWholeTask(1f / numOfSnapshots);
+ int res = 0;
+ String[] args = new String[4];
+ args[0] = "-snapshot";
+ args[1] = backupContext.getSnapshotName(table);
+ args[2] = "-copy-to";
+ args[3] = backupContext.getBackupStatus(table).getTargetDir();
+
+ LOG.debug("Copy snapshot " + args[1] + " to " + args[3]);
+ res = copyService.copy(this, conf, BackupCopyService.Type.FULL, args);
+ // if one snapshot export failed, do not continue for remained snapshots
+ if (res != 0) {
+
+ LOG.error("Exporting Snapshot " + args[1] + " failed with return code: " + res + ".");
+
+ throw new IOException("Failed of exporting snapshot " + args[1] + " to " + args[3]
+ + " with reason code " + res);
+ }
+
+ LOG.info("Snapshot copy " + args[1] + " finished.");
+ }
+ }
+
+ /**
+ * Wrap a SnapshotDescription for a target table.
+ * @param table table
+ * @return a SnapshotDescription especially for backup.
+ */
+ private SnapshotDescription wrapSnapshotDescription(String table) {
+ // Mock a SnapshotDescription from backupContext to call SnapshotManager function,
+ // Name it in the format "snapshot_<timestamp>_<table>"
+ HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder();
+ builder.setTable(table);
+ TableName tableName = TableName.valueOf(table);
+ builder.setName("snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime()) + "_"
+ + tableName.getNamespaceAsString() + "_" + tableName.getQualifierAsString());
+ HBaseProtos.SnapshotDescription backupSnapshot = builder.build();
+
+ LOG.debug("Wrapped a SnapshotDescription " + backupSnapshot.getName()
+ + " from backupContext to request snapshot for backup.");
+
+ return backupSnapshot;
+ }
+
+ /**
+ * Delete HBase snapshot for backup.
+ * @param backupCtx backup context
+ * @throws Exception exception
+ */
+ private void deleteSnapshot(BackupContext backupCtx) throws IOException {
+
+ LOG.debug("Trying to delete snapshot for full backup.");
+ Connection conn = null;
+ Admin admin = null;
+ try {
+ conn = ConnectionFactory.createConnection(conf);
+ admin = conn.getAdmin();
+ for (String snapshotName : backupCtx.getSnapshotNames()) {
+ if (snapshotName == null) {
+ continue;
+ }
+ LOG.debug("Trying to delete snapshot: " + snapshotName);
+ admin.deleteSnapshot(snapshotName);
+ LOG.debug("Deleting the snapshot " + snapshotName + " for backup "
+ + backupCtx.getBackupId() + " succeeded.");
+ }
+ } finally {
+ if (admin != null) {
+ admin.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ /**
+ * Clean up directories with prefix "exportSnapshot-", which are generated when exporting
+ * snapshots.
+ * @throws IOException exception
+ */
+ private void cleanupExportSnapshotLog() throws IOException {
+ FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+ Path stagingDir =
+ new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory()
+ .toString()));
+ FileStatus[] files = FSUtils.listStatus(fs, stagingDir);
+ if (files == null) {
+ return;
+ }
+ for (FileStatus file : files) {
+ if (file.getPath().getName().startsWith("exportSnapshot-")) {
+ LOG.debug("Delete log files of exporting snapshot: " + file.getPath().getName());
+ if (FSUtils.delete(fs, file.getPath(), true) == false) {
+ LOG.warn("Can not delete " + file.getPath());
+ }
+ }
+ }
+ }
+
+ /**
+ * Clean up directories with prefix "_distcp_logs-", which are generated when DistCp copying
+ * hlogs.
+ * @throws IOException exception
+ */
+ private void cleanupDistCpLog() throws IOException {
+ Path rootPath = new Path(backupContext.getHLogTargetDir()).getParent();
+ FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
+ FileStatus[] files = FSUtils.listStatus(fs, rootPath);
+ if (files == null) {
+ return;
+ }
+ for (FileStatus file : files) {
+ if (file.getPath().getName().startsWith("_distcp_logs")) {
+ LOG.debug("Delete log files of DistCp: " + file.getPath().getName());
+ FSUtils.delete(fs, file.getPath(), true);
+ }
+ }
+ }
+
+ /**
+ * Clean up the uncompleted data at target directory if the ongoing backup has already entered the
+ * copy phase.
+ */
+ private void cleanupTargetDir() {
+ try {
+ // clean up the uncompleted data at target directory if the ongoing backup has already entered
+ // the copy phase
+ LOG.debug("Trying to cleanup up target dir. Current backup phase: "
+ + backupContext.getPhase());
+ if (backupContext.getPhase().equals(BACKUPPHASE.SNAPSHOTCOPY)
+ || backupContext.getPhase().equals(BACKUPPHASE.INCREMENTAL_COPY)
+ || backupContext.getPhase().equals(BACKUPPHASE.STORE_MANIFEST)) {
+ FileSystem outputFs =
+ FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf);
+
+ // now treat one backup as a transaction, clean up data that has been partially copied at
+ // table level
+ for (String table : backupContext.getTables()) {
+ Path targetDirPath =
+ new Path(HBackupFileSystem.getTableBackupDir(backupContext.getTargetRootDir(),
+ backupContext.getBackupId(), table));
+ if (outputFs.delete(targetDirPath, true)) {
+ LOG.info("Cleaning up uncompleted backup data at " + targetDirPath.toString()
+ + " done.");
+ } else {
+ LOG.info("No data has been copied to " + targetDirPath.toString() + ".");
+ }
+
+ Path tableDir = targetDirPath.getParent();
+ FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir);
+ if (backups == null || backups.length == 0) {
+ outputFs.delete(tableDir, true);
+ LOG.debug(tableDir.toString() + " is empty, remove it.");
+ }
+ }
+ }
+
+ } catch (IOException e1) {
+ LOG.error("Cleaning up uncompleted backup data of " + backupContext.getBackupId() + " at "
+ + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
+ }
+ }
+
+ /**
+ * Add manifest for the current backup. The manifest is stored
+ * within the table backup directory.
+ * @param backupContext The current backup context
+ * @throws IOException exception
+ * @throws BackupException exception
+ */
+ private void addManifest(BackupContext backupContext) throws IOException, BackupException {
+ // set the overall backup phase : store manifest
+ backupContext.setPhase(BACKUPPHASE.STORE_MANIFEST);
+
+ // avoid action if has been cancelled
+ if (backupContext.isCancelled()) {
+ return;
+ }
+
+ BackupManifest manifest;
+ boolean fromExistingSnapshot = false; // to be implemented in future jira
+
+ // Since we have each table's backup in its own directory structure,
+ // we'll store its manifest with the table directory.
+ for (String table : backupContext.getTables()) {
+ manifest = new BackupManifest(backupContext, table);
+ if (fromExistingSnapshot) {
+ // mark backing up from existing snapshot in manifest, so that later, dependency analysis
+ // can skip this backup image
+ LOG.debug("backup using existing snapshot will be supported in future jira");
+ } else {
+ ArrayList<BackupImage> ancestors = this.backupManager.getAncestors(backupContext, table);
+ for (BackupImage image : ancestors) {
+ manifest.addDependentImage(image);
+ }
+ }
+ if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ // We'll store the log timestamps for this table only in its manifest.
+ HashMap<String, HashMap<String, String>> tableTimestampMap =
+ new HashMap<String, HashMap<String, String>>();
+ tableTimestampMap.put(table, backupContext.getIncrTimestampMap().get(table));
+ manifest.setIncrTimestampMap(tableTimestampMap);
+ }
+ manifest.store(conf);
+ }
+
+ // For incremental backup, we store a overall manifest in
+ // <backup-root-dir>/WALs/<backup-id>
+ // This is used when created the next incremental backup
+ if (backupContext.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ manifest = new BackupManifest(backupContext);
+ // set the table region server start and end timestamps for incremental backup
+ manifest.setIncrTimestampMap(backupContext.getIncrTimestampMap());
+ ArrayList<BackupImage> ancestors = this.backupManager.getAncestors(backupContext);
+ for (BackupImage image : ancestors) {
+ manifest.addDependentImage(image);
+ }
+ manifest.store(conf);
+ }
+ }
+
+ /**
+ * Do incremental copy.
+ * @param backupContext backup context
+ */
+ private void incrementalCopy(BackupContext backupContext) throws Exception {
+
+ LOG.info("Incremental copy is starting.");
+
+ // set overall backup phase: incremental_copy
+ backupContext.setPhase(BACKUPPHASE.INCREMENTAL_COPY);
+
+ // avoid action if has been cancelled
+ if (backupContext.isCancelled()) {
+ return;
+ }
+
+ // get incremental backup file list and prepare parms for DistCp
+ List<String> incrBackupFileList = backupContext.getIncrBackupFileList();
+ String[] strArr = incrBackupFileList.toArray(new String[incrBackupFileList.size() + 1]);
+ strArr[strArr.length - 1] = backupContext.getHLogTargetDir();
+
+ BackupCopyService copyService = BackupRestoreServiceFactory.getBackupCopyService(conf);
+ int res = copyService.copy(this, conf, BackupCopyService.Type.INCREMENTAL, strArr);
+
+ if (res != 0) {
+ LOG.error("Copy incremental log files failed with return code: " + res + ".");
+ throw new IOException("Failed of Hadoop Distributed Copy from " + incrBackupFileList + " to "
+ + backupContext.getHLogTargetDir());
+ }
+ LOG.info("Incremental copy from " + incrBackupFileList + " to "
+ + backupContext.getHLogTargetDir() + " finished.");
+
+ }
+
+ private String getMessage(Exception e) {
+ String msg = e.getMessage();
+ if (msg == null || msg.equals("")) {
+ msg = e.getClass().getName();
+ }
+ return msg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java
new file mode 100644
index 0000000..fdb3c46
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManager.java
@@ -0,0 +1,488 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS;
+import org.apache.hadoop.hbase.backup.BackupManifest.BackupImage;
+import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData;
+import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+
+
+/**
+ * Handles backup requests on server-side, creates backup context records in hbase:backup
+ * to keep track backup. The timestamps kept in hbase:backup table will be used for future
+ * incremental backup. Creates BackupContext and DispatchRequest.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupManager {
+ private static final Log LOG = LogFactory.getLog(BackupManager.class);
+
+ private Configuration conf = null;
+ private BackupContext backupContext = null;
+ private ExecutorService pool = null;
+
+ private boolean backupComplete = false;
+
+ private BackupSystemTable systemTable;
+
+ /**
+ * Backup manager constructor.
+ * @param conf configuration
+ * @throws IOException exception
+ */
+ public BackupManager(Configuration conf) throws IOException {
+ if (!conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT)) {
+ throw new BackupException("HBase backup is not enabled. Check your " +
+ HConstants.BACKUP_ENABLE_KEY + " setting.");
+ }
+ this.conf = conf;
+ this.systemTable = BackupSystemTable.getTable(conf);
+ Runtime.getRuntime().addShutdownHook(new ExitHandler());
+ }
+
+ /**
+ * This method modifies the master's configuration in order to inject backup-related features
+ * @param conf configuration
+ */
+ public static void decorateMasterConfiguration(Configuration conf) {
+ if (!isBackupEnabled(conf)) {
+ return;
+ }
+ String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
+ String cleanerClass = BackupLogCleaner.class.getCanonicalName();
+ if (!plugins.contains(cleanerClass)) {
+ conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Added log cleaner: " + cleanerClass);
+ }
+ }
+
+ private static boolean isBackupEnabled(Configuration conf) {
+ return conf.getBoolean(HConstants.BACKUP_ENABLE_KEY, HConstants.BACKUP_ENABLE_DEFAULT);
+ }
+
+ private class ExitHandler extends Thread {
+ public ExitHandler() {
+ super("Backup Manager Exit Handler");
+ }
+
+ public void run() {
+
+ if (backupContext != null && !backupComplete) {
+
+ // program exit and backup is not complete, then mark as cancelled to avoid submitted backup
+ // handler's taking further action
+ backupContext.markCancel();
+
+ LOG.debug("Backup is cancelled due to force program exiting.");
+ try {
+ cancelBackup(backupContext.getBackupId());
+ } catch (Exception e) {
+ String msg = e.getMessage();
+ if (msg == null || msg.equals("")) {
+ msg = e.getClass().getName();
+ }
+ LOG.error("Failed to cancel backup " + backupContext.getBackupId() + " due to " + msg);
+ }
+ }
+
+ exit();
+ }
+ }
+
+ /**
+ * Cancel the ongoing backup via backup id.
+ * @param backupId The id of the ongoing backup to be cancelled
+ * @throws Exception exception
+ */
+ private void cancelBackup(String backupId) throws Exception {
+ // TODO: will be implemented in Phase 2: HBASE-14125
+ LOG.debug("Try to cancel the backup " + backupId + ". the feature is NOT implemented yet");
+
+ }
+
+ /**
+ * Stop all the work of backup.
+ */
+ public void exit() {
+
+ // currently, we shutdown now for all ongoing back handlers, we may need to do something like
+ // record the failed list somewhere later
+ if (this.pool != null) {
+ this.pool.shutdownNow();
+ }
+
+ }
+
+ /**
+ * Create a BackupContext based on input backup request.
+ * @param backupId backup id
+ * @param type type
+ * @param tablelist table list
+ * @param targetRootDir root dir
+ * @param snapshot snapshot name
+ * @return BackupContext context
+ * @throws BackupException exception
+ */
+ protected BackupContext createBackupContext(String backupId, String type, String tablelist,
+ String targetRootDir, String snapshot) throws BackupException {
+
+ if (targetRootDir == null) {
+ throw new BackupException("Wrong backup request parameter: target backup root directory");
+ }
+
+ if (type.equals(BackupRestoreConstants.BACKUP_TYPE_FULL) && tablelist == null) {
+ // If table list is null for full backup, which means backup all tables. Then fill the table
+ // list with all user tables from meta. It no table available, throw the request exception.
+
+ HTableDescriptor[] htds = null;
+ try (Connection conn = ConnectionFactory.createConnection(conf);
+ HBaseAdmin hbadmin = (HBaseAdmin) conn.getAdmin()) {
+
+ htds = hbadmin.listTables();
+ } catch (Exception e) {
+ throw new BackupException(e);
+ }
+
+ if (htds == null) {
+ throw new BackupException("No table exists for full backup of all tables.");
+ } else {
+ StringBuilder sb = new StringBuilder();
+ for (HTableDescriptor hTableDescriptor : htds) {
+ sb.append(hTableDescriptor.getNameAsString()
+ + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+ }
+ sb.deleteCharAt(sb.lastIndexOf(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND));
+ tablelist = sb.toString();
+
+ LOG.info("Full backup all the tables available in the cluster: " + tablelist);
+ }
+ }
+
+ // there are one or more tables in the table list
+ return new BackupContext(backupId, type,
+ tablelist.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND), targetRootDir,
+ snapshot);
+
+ }
+
+ /**
+ * Check if any ongoing backup. Currently, we only reply on checking status in hbase:backup. We
+ * need to consider to handle the case of orphan records in the future. Otherwise, all the coming
+ * request will fail.
+ * @return the ongoing backup id if on going backup exists, otherwise null
+ * @throws IOException exception
+ */
+ private String getOngoingBackupId() throws IOException {
+
+ ArrayList<BackupContext> sessions = systemTable.getBackupContexts(BACKUPSTATUS.ONGOING);
+ if (sessions.size() == 0) {
+ return null;
+ }
+ return sessions.get(0).getBackupId();
+ }
+
+ /**
+ * Start the backup manager service.
+ * @throws IOException exception
+ */
+ public void initialize() throws IOException {
+ String ongoingBackupId = this.getOngoingBackupId();
+ if (ongoingBackupId != null) {
+ LOG.info("There is a ongoing backup " + ongoingBackupId
+ + ". Can not launch new backup until no ongoing backup remains.");
+ throw new BackupException("There is ongoing backup.");
+ }
+
+ // Initialize thread pools
+ int nrThreads = this.conf.getInt("hbase.backup.threads.max", 1);
+ ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+ builder.setNameFormat("BackupHandler-%1$d");
+ this.pool =
+ new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), builder.build());
+ ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
+ }
+
+ /**
+ * Dispatch and handle a backup request.
+ * @param backupContext backup context
+ * @throws BackupException exception
+ */
+ public void dispatchRequest(BackupContext backupContext) throws BackupException {
+
+ this.backupContext = backupContext;
+
+ LOG.info("Got a backup request: " + "Type: " + backupContext.getType() + "; Tables: "
+ + backupContext.getTableListAsString() + "; TargetRootDir: "
+ + backupContext.getTargetRootDir());
+
+ // dispatch the request to a backup handler and put it handler map
+
+ BackupHandler handler = new BackupHandler(this.backupContext, this, conf);
+ Future<Object> future = this.pool.submit(handler);
+ // wait for the execution to complete
+ try {
+ future.get();
+ } catch (InterruptedException e) {
+ throw new BackupException(e);
+ } catch (CancellationException e) {
+ throw new BackupException(e);
+ } catch (ExecutionException e) {
+ throw new BackupException(e);
+ }
+
+ // mark the backup complete for exit handler's processing
+ backupComplete = true;
+
+ LOG.info("Backup request " + backupContext.getBackupId() + " has been executed.");
+ }
+
+ /**
+ * Get direct ancestors of the current backup.
+ * @param backupCtx The backup context for the current backup
+ * @return The ancestors for the current backup
+ * @throws IOException exception
+ * @throws BackupException exception
+ */
+ protected ArrayList<BackupImage> getAncestors(BackupContext backupCtx) throws IOException,
+ BackupException {
+ LOG.debug("Getting the direct ancestors of the current backup ...");
+
+ ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>();
+
+ // full backup does not have ancestor
+ if (backupCtx.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+ LOG.debug("Current backup is a full backup, no direct ancestor for it.");
+ return ancestors;
+ }
+
+ // get all backup history list in descending order
+
+ ArrayList<BackupCompleteData> allHistoryList = getBackupHistory();
+ for (BackupCompleteData backup : allHistoryList) {
+ BackupImage image =
+ new BackupImage(backup.getBackupToken(), backup.getType(), backup.getBackupRootPath(),
+ backup.getTableList(), Long.parseLong(backup.getStartTime()), Long.parseLong(backup
+ .getEndTime()));
+
+ // add the full backup image as an ancestor until the last incremental backup
+ if (backup.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+
+ // backup image from existing snapshot does not involve in dependency
+ if (backup.fromExistingSnapshot()) {
+ continue;
+ }
+ // check the backup image coverage, if previous image could be covered by the newer ones,
+ // then no need to add
+ if (!BackupManifest.canCoverImage(ancestors, image)) {
+ ancestors.add(image);
+ }
+ } else {
+ // found last incremental backup, if previously added full backup ancestor images can cover
+ // it, then this incremental ancestor is not the dependent of the current incremental
+ // backup, that is to say, this is the backup scope boundary of current table set.
+ // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing
+ // incremental backup
+ if (BackupManifest.canCoverImage(ancestors, image)) {
+ LOG.debug("Met the backup boundary of the current table set. "
+ + "The root full backup images for the current backup scope:");
+ for (BackupImage image1 : ancestors) {
+ LOG.debug(" BackupId: " + image1.getBackupId() + ", Backup directory: "
+ + image1.getRootDir());
+ }
+ } else {
+ Path logBackupPath =
+ HBackupFileSystem.getLogBackupPath(backup.getBackupRootPath(),
+ backup.getBackupToken());
+ LOG.debug("Current backup has an incremental backup ancestor, "
+ + "touching its image manifest in " + logBackupPath.toString()
+ + " to construct the dependency.");
+
+ BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath);
+ BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage();
+ ancestors.add(lastIncrImage);
+
+ LOG.debug("Last dependent incremental backup image information:");
+ LOG.debug(" Token: " + lastIncrImage.getBackupId());
+ LOG.debug(" Backup directory: " + lastIncrImage.getRootDir());
+ }
+ }
+ }
+ LOG.debug("Got " + ancestors.size() + " ancestors for the current backup.");
+ return ancestors;
+ }
+
+ /**
+ * Get the direct ancestors of this backup for one table involved.
+ * @param backupContext backup context
+ * @param table table
+ * @return backupImages on the dependency list
+ * @throws BackupException exception
+ * @throws IOException exception
+ */
+ protected ArrayList<BackupImage> getAncestors(BackupContext backupContext, String table)
+ throws BackupException, IOException {
+ ArrayList<BackupImage> ancestors = getAncestors(backupContext);
+ ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>();
+ for (BackupImage image : ancestors) {
+ if (image.hasTable(table)) {
+ tableAncestors.add(image);
+ if (image.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+ break;
+ }
+ }
+ }
+ return tableAncestors;
+ }
+
+ /**
+ * hbase:backup operations
+ */
+
+ /**
+ * Updates status (state) of a backup session in a persistent store
+ * @param context context
+ * @throws IOException exception
+ */
+ public void updateBackupStatus(BackupContext context) throws IOException {
+ systemTable.updateBackupStatus(context);
+ }
+
+ /**
+ * Read the last backup start code (timestamp) of last successful backup. Will return null
+ * if there is no startcode stored in hbase:backup or the value is of length 0. These two
+ * cases indicate there is no successful backup completed so far.
+ * @return the timestamp of a last successful backup
+ * @throws IOException exception
+ */
+ public String readBackupStartCode() throws IOException {
+ return systemTable.readBackupStartCode();
+ }
+
+ /**
+ * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte.
+ * @param startCode start code
+ * @throws IOException exception
+ */
+ public void writeBackupStartCode(String startCode) throws IOException {
+ systemTable.writeBackupStartCode(startCode);
+ }
+
+ /**
+ * Get the RS log information after the last log roll from hbase:backup.
+ * @return RS log info
+ * @throws IOException exception
+ */
+ public HashMap<String, String> readRegionServerLastLogRollResult() throws IOException {
+ return systemTable.readRegionServerLastLogRollResult();
+ }
+
+ /**
+ * Get all completed backup information (in desc order by time)
+ * @return history info of BackupCompleteData
+ * @throws IOException exception
+ */
+ public ArrayList<BackupCompleteData> getBackupHistory() throws IOException {
+ return systemTable.getBackupHistory();
+ }
+
+ /**
+ * Write the current timestamps for each regionserver to hbase:backup after a successful full or
+ * incremental backup. Each table may have a different set of log timestamps. The saved timestamp
+ * is of the last log file that was backed up already.
+ * @param tables tables
+ * @throws IOException exception
+ */
+ public void writeRegionServerLogTimestamp(Set<String> tables,
+ HashMap<String, String> newTimestamps) throws IOException {
+ systemTable.writeRegionServerLogTimestamp(tables, newTimestamps);
+ }
+
+ /**
+ * Read the timestamp for each region server log after the last successful backup. Each table has
+ * its own set of the timestamps. The info is stored for each table as a concatinated string on ZK
+ * under //hbase//backup//incr//tablelogtimestamp//table_name
+ * @return the timestamp for each region server. key: tableName value:
+ * RegionServer,PreviousTimeStamp
+ * @throws IOException exception
+ */
+ public HashMap<String, HashMap<String, String>> readLogTimestampMap() throws IOException {
+ return systemTable.readLogTimestampMap();
+ }
+
+ /**
+ * Return the current tables covered by incremental backup.
+ * @return set of tableNames
+ * @throws IOException exception
+ */
+ public Set<String> getIncrementalBackupTableSet() throws IOException {
+ return systemTable.getIncrementalBackupTableSet();
+ }
+
+ /**
+ * Adds set of tables to overall incremental backup table set
+ * @param tables tables
+ * @throws IOException exception
+ */
+ public void addIncrementalBackupTableSet(Set<String> tables) throws IOException {
+ systemTable.addIncrementalBackupTableSet(tables);
+ }
+
+ /**
+ * Saves list of WAL files after incremental backup operation. These files will be stored until
+ * TTL expiration and are used by Backup Log Cleaner plugin to determine which WAL files can be
+ * safely purged.
+ */
+
+ public void recordWALFiles(List<String> files) throws IOException {
+ systemTable.addWALFiles(files, backupContext.getBackupId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java
new file mode 100644
index 0000000..f41540b
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupManifest.java
@@ -0,0 +1,814 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * Backup manifest Contains all the meta data of a backup image. The manifest info will be bundled
+ * as manifest file together with data. So that each backup image will contain all the info needed
+ * for restore.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupManifest {
+
+ private static final Log LOG = LogFactory.getLog(BackupManifest.class);
+
+ // manifest file name
+ public static final String FILE_NAME = ".backup.manifest";
+
+ // manifest file version, current is 1.0
+ public static final String MANIFEST_VERSION = "1.0";
+
+ // tags of fields for manifest file
+ public static final String TAG_VERSION = "Manifest-Version";
+ public static final String TAG_BACKUPID = "Backup-Id";
+ public static final String TAG_BACKUPTYPE = "Backup-Type";
+ public static final String TAG_TABLESET = "Table-Set";
+ public static final String TAG_STARTTS = "Start-Timestamp";
+ public static final String TAG_COMPLETETS = "Complete-Timestamp";
+ public static final String TAG_TABLEBYTES = "Total-Table-Bytes";
+ public static final String TAG_LOGBYTES = "Total-Log-Bytes";
+ public static final String TAG_INCRTIMERANGE = "Incremental-Time-Range";
+ public static final String TAG_DEPENDENCY = "Dependency";
+ public static final String TAG_IMAGESTATE = "Image-State";
+ public static final String TAG_COMPACTION = "Compaction";
+
+ public static final String ERROR_DEPENDENCY = "DEPENDENCY_ERROR";
+
+ public static final int DELETE_SUCCESS = 0;
+ public static final int DELETE_FAILED = -1;
+
+ // currently only one state, will have CONVERTED, and MERGED in future JIRA
+ public static final String IMAGE_STATE_ORIG = "ORIGINAL";
+ public static final String IMAGE_STATE_CONVERT = "CONVERTED";
+ public static final String IMAGE_STATE_MERGE = "MERGED";
+ public static final String IMAGE_STATE_CONVERT_MERGE = "CONVERTED,MERGED";
+
+ // backup image, the dependency graph is made up by series of backup images
+
+ public static class BackupImage implements Comparable<BackupImage> {
+
+ private String backupId;
+ private String type;
+ private String rootDir;
+ private String tableSet;
+ private long startTs;
+ private long completeTs;
+ private ArrayList<BackupImage> ancestors;
+
+ public BackupImage() {
+ super();
+ }
+
+ public BackupImage(String backupId, String type, String rootDir, String tableSet, long startTs,
+ long completeTs) {
+ this.backupId = backupId;
+ this.type = type;
+ this.rootDir = rootDir;
+ this.tableSet = tableSet;
+ this.startTs = startTs;
+ this.completeTs = completeTs;
+ }
+
+ public String getBackupId() {
+ return backupId;
+ }
+
+ public void setBackupId(String backupId) {
+ this.backupId = backupId;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getRootDir() {
+ return rootDir;
+ }
+
+ public void setRootDir(String rootDir) {
+ this.rootDir = rootDir;
+ }
+
+ public String getTableSet() {
+ return tableSet;
+ }
+
+ public void setTableSet(String tableSet) {
+ this.tableSet = tableSet;
+ }
+
+ public long getStartTs() {
+ return startTs;
+ }
+
+ public void setStartTs(long startTs) {
+ this.startTs = startTs;
+ }
+
+ public long getCompleteTs() {
+ return completeTs;
+ }
+
+ public void setCompleteTs(long completeTs) {
+ this.completeTs = completeTs;
+ }
+
+ public ArrayList<BackupImage> getAncestors() {
+ if (this.ancestors == null) {
+ this.ancestors = new ArrayList<BackupImage>();
+ }
+ return this.ancestors;
+ }
+
+ public void addAncestor(BackupImage backupImage) {
+ this.getAncestors().add(backupImage);
+ }
+
+ public boolean hasAncestor(String token) {
+ for (BackupImage image : this.getAncestors()) {
+ if (image.getBackupId().equals(token)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean hasTable(String table) {
+ String[] tables = this.getTableSet().split(";");
+ for (String t : tables) {
+ if (t.equals(table)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(BackupImage other) {
+ String thisBackupId = this.getBackupId();
+ String otherBackupId = other.getBackupId();
+ Long thisTS = new Long(thisBackupId.substring(thisBackupId.lastIndexOf("_") + 1));
+ Long otherTS = new Long(otherBackupId.substring(otherBackupId.lastIndexOf("_") + 1));
+ return thisTS.compareTo(otherTS);
+ }
+ }
+
+ // manifest version
+ private String version = MANIFEST_VERSION;
+
+ // hadoop hbase configuration
+ protected Configuration config = null;
+
+ // backup root directory
+ private String rootDir = null;
+
+ // backup image directory
+ private String tableBackupDir = null;
+
+ // backup log directory if this is an incremental backup
+ private String logBackupDir = null;
+
+ // backup token
+ private String token;
+
+ // backup type, full or incremental
+ private String type;
+
+ // the table set for the backup
+ private ArrayList<String> tableSet;
+
+ // actual start timestamp of the backup process
+ private long startTs;
+
+ // actual complete timestamp of the backup process
+ private long completeTs;
+
+ // total bytes for table backup image
+ private long tableBytes;
+
+ // total bytes for the backed-up logs for incremental backup
+ private long logBytes;
+
+ // the region server timestamp for tables:
+ // <table, <rs, timestamp>>
+ private Map<String, HashMap<String, String>> incrTimeRanges;
+
+ // dependency of this backup, including all the dependent images to do PIT recovery
+ private Map<String, BackupImage> dependency;
+
+ // the state of backup image
+ private String imageState;
+
+ // the indicator of the image compaction
+ private boolean isCompacted = false;
+
+ // the merge chain of the original backups, null if not a merged backup
+ private LinkedList<String> mergeChain;
+
+ /**
+ * Construct manifest for a ongoing backup.
+ * @param backupCtx The ongoing backup context
+ */
+ public BackupManifest(BackupContext backupCtx) {
+ this.token = backupCtx.getBackupId();
+ this.type = backupCtx.getType();
+ this.rootDir = backupCtx.getTargetRootDir();
+ if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ this.logBackupDir = backupCtx.getHLogTargetDir();
+ this.logBytes = backupCtx.getTotalBytesCopied();
+ }
+ this.startTs = backupCtx.getStartTs();
+ this.completeTs = backupCtx.getEndTs();
+ this.loadTableSet(backupCtx.getTableListAsString());
+ this.setImageOriginal();
+ }
+
+ /**
+ * Construct a table level manifest for a backup of the named table.
+ * @param backupCtx The ongoing backup context
+ */
+ public BackupManifest(BackupContext backupCtx, String table) {
+ this.token = backupCtx.getBackupId();
+ this.type = backupCtx.getType();
+ this.rootDir = backupCtx.getTargetRootDir();
+ this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir();
+ if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ this.logBackupDir = backupCtx.getHLogTargetDir();
+ this.logBytes = backupCtx.getTotalBytesCopied();
+ }
+ this.startTs = backupCtx.getStartTs();
+ this.completeTs = backupCtx.getEndTs();
+ this.loadTableSet(table);
+ this.setImageOriginal();
+ }
+
+ /**
+ * Construct manifest from a backup directory.
+ * @param conf configuration
+ * @param backupPath backup path
+ * @throws BackupException exception
+ */
+ public BackupManifest(Configuration conf, Path backupPath) throws BackupException {
+
+ LOG.debug("Loading manifest from: " + backupPath.toString());
+ // The input backupDir may not exactly be the backup table dir.
+ // It could be the backup log dir where there is also a manifest file stored.
+ // This variable's purpose is to keep the correct and original location so
+ // that we can store/persist it.
+ this.tableBackupDir = backupPath.toString();
+ this.config = conf;
+ try {
+
+ FileSystem fs = backupPath.getFileSystem(conf);
+ FileStatus[] subFiles = FSUtils.listStatus(fs, backupPath);
+ if (subFiles == null) {
+ String errorMsg = backupPath.toString() + " does not exist";
+ LOG.error(errorMsg);
+ throw new IOException(errorMsg);
+ }
+ for (FileStatus subFile : subFiles) {
+ if (subFile.getPath().getName().equals(FILE_NAME)) {
+
+ // load and set manifest field from file content
+ FSDataInputStream in = fs.open(subFile.getPath());
+ Properties props = new Properties();
+ try {
+ props.load(in);
+ } catch (IOException e) {
+ LOG.error("Error when loading from manifest file!");
+ throw e;
+ } finally {
+ in.close();
+ }
+
+ this.version = props.getProperty(TAG_VERSION);
+ this.token = props.getProperty(TAG_BACKUPID);
+ this.type = props.getProperty(TAG_BACKUPTYPE);
+ // Here the parameter backupDir is where the manifest file is.
+ // There should always be a manifest file under:
+ // backupRootDir/namespace/table/backupId/.backup.manifest
+ this.rootDir = backupPath.getParent().getParent().getParent().toString();
+
+ Path p = backupPath.getParent();
+ if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
+ this.rootDir = p.getParent().toString();
+ } else {
+ this.rootDir = p.getParent().getParent().toString();
+ }
+
+ this.loadTableSet(props.getProperty(TAG_TABLESET));
+
+ this.startTs = Long.parseLong(props.getProperty(TAG_STARTTS));
+ this.completeTs = Long.parseLong(props.getProperty(TAG_COMPLETETS));
+ this.tableBytes = Long.parseLong(props.getProperty(TAG_TABLEBYTES));
+ if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ this.logBytes = (Long.parseLong(props.getProperty(TAG_LOGBYTES)));
+ LOG.debug("convert will be implemented by future jira");
+ }
+ this.loadIncrementalTimeRanges(props.getProperty(TAG_INCRTIMERANGE));
+ this.loadDependency(props.getProperty(TAG_DEPENDENCY));
+ this.imageState = props.getProperty(TAG_IMAGESTATE);
+ this.isCompacted =
+ props.getProperty(TAG_COMPACTION).equalsIgnoreCase("TRUE") ? true : false;
+ LOG.debug("merge and from existing snapshot will be implemented by future jira");
+ LOG.debug("Loaded manifest instance from manifest file: "
+ + FSUtils.getPath(subFile.getPath()));
+ return;
+ }
+ }
+ String errorMsg = "No manifest file found in: " + backupPath.toString();
+ LOG.error(errorMsg);
+ throw new IOException(errorMsg);
+
+ } catch (IOException e) {
+ throw new BackupException(e.getMessage());
+ }
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ /**
+ * Load table set from a table set list string (t1;t2;t3;...).
+ * @param tableSetStr Table set list string
+ */
+ private void loadTableSet(String tableSetStr) {
+
+ LOG.debug("Loading table set: " + tableSetStr);
+
+ String[] tableSet = tableSetStr.split(";");
+ this.tableSet = this.getTableSet();
+ if (this.tableSet.size() > 0) {
+ this.tableSet.clear();
+ }
+ for (int i = 0; i < tableSet.length; i++) {
+ this.tableSet.add(tableSet[i]);
+ }
+
+ LOG.debug(tableSet.length + " tables exist in table set.");
+ }
+
+ public void setImageOriginal() {
+ this.imageState = IMAGE_STATE_ORIG;
+ }
+
+ /**
+ * Get the table set of this image.
+ * @return The table set list
+ */
+ public ArrayList<String> getTableSet() {
+ if (this.tableSet == null) {
+ this.tableSet = new ArrayList<String>();
+ }
+ return this.tableSet;
+ }
+
+ /**
+ * Persist the manifest file.
+ * @throws IOException IOException when storing the manifest file.
+ */
+ public void store(Configuration conf) throws BackupException {
+ Properties props = new Properties();
+ props.setProperty(TAG_VERSION, this.version);
+ props.setProperty(TAG_BACKUPID, this.token);
+ props.setProperty(TAG_BACKUPTYPE, this.type);
+ props.setProperty(TAG_TABLESET, this.getTableSetStr());
+ LOG.debug("convert will be supported in future jira");
+ // String convertedTables = this.getConvertedTableSetStr();
+ // if (convertedTables != null )
+ // props.setProperty(TAG_CONVERTEDTABLESET, convertedTables);
+ props.setProperty(TAG_STARTTS, Long.toString(this.startTs));
+ props.setProperty(TAG_COMPLETETS, Long.toString(this.completeTs));
+ props.setProperty(TAG_TABLEBYTES, Long.toString(this.tableBytes));
+ if (this.type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ props.setProperty(TAG_LOGBYTES, Long.toString(this.logBytes));
+ }
+ props.setProperty(TAG_INCRTIMERANGE, this.getIncrTimestampStr());
+ props.setProperty(TAG_DEPENDENCY, this.getDependencyStr());
+ props.setProperty(TAG_IMAGESTATE, this.getImageState());
+ props.setProperty(TAG_COMPACTION, this.isCompacted ? "TRUE" : "FALSE");
+ LOG.debug("merge will be supported in future jira");
+ // props.setProperty(TAG_MERGECHAIN, this.getMergeChainStr());
+ LOG.debug("backup from existing snapshot will be supported in future jira");
+ // props.setProperty(TAG_FROMSNAPSHOT, this.isFromSnapshot() ? "TRUE" : "FALSE");
+
+ // write the file, overwrite if already exist
+ Path manifestFilePath =
+ new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir)
+ + File.separator + FILE_NAME);
+ try {
+ FSDataOutputStream out = manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);
+ props.store(out, "HBase backup manifest.");
+ out.close();
+ } catch (IOException e) {
+ throw new BackupException(e.getMessage());
+ }
+
+ LOG.debug("Manifest file stored to " + this.tableBackupDir != null ? this.tableBackupDir
+ : this.logBackupDir + File.separator + FILE_NAME);
+ }
+
+ /**
+ * Get the table set string in the format of t1;t2;t3...
+ */
+ private String getTableSetStr() {
+ return BackupUtil.concat(getTableSet(), ";");
+ }
+
+ public String getImageState() {
+ return imageState;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ /**
+ * Get this backup image.
+ * @return the backup image.
+ */
+ public BackupImage getBackupImage() {
+ return this.getDependency().get(this.token);
+ }
+
+ /**
+ * Add dependent backup image for this backup.
+ * @param image The direct dependent backup image
+ */
+ public void addDependentImage(BackupImage image) {
+ this.getDependency().get(this.token).addAncestor(image);
+ this.setDependencyMap(this.getDependency(), image);
+ }
+
+ /**
+ * Get the dependency' string in the json format.
+ */
+ private String getDependencyStr() {
+ BackupImage thisImage = this.getDependency().get(this.token);
+ if (thisImage == null) {
+ LOG.warn("There is no dependency set yet.");
+ return null;
+ }
+
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ return mapper.writeValueAsString(thisImage);
+ } catch (JsonGenerationException e) {
+ LOG.error("Error when generating dependency string from backup image.", e);
+ return ERROR_DEPENDENCY;
+ } catch (JsonMappingException e) {
+ LOG.error("Error when generating dependency string from backup image.", e);
+ return ERROR_DEPENDENCY;
+ } catch (IOException e) {
+ LOG.error("Error when generating dependency string from backup image.", e);
+ return ERROR_DEPENDENCY;
+ }
+ }
+
+ /**
+ * Get all dependent backup images. The image of this backup is also contained.
+ * @return The dependent backup images map
+ */
+ public Map<String, BackupImage> getDependency() {
+ if (this.dependency == null) {
+ this.dependency = new HashMap<String, BackupImage>();
+ LOG.debug(this.rootDir + " " + this.token + " " + this.type);
+ this.dependency.put(this.token,
+ new BackupImage(this.token, this.type, this.rootDir, this.getTableSetStr(), this.startTs,
+ this.completeTs));
+ }
+ return this.dependency;
+ }
+
+ /**
+ * Set the incremental timestamp map directly.
+ * @param incrTimestampMap timestamp map
+ */
+ public void setIncrTimestampMap(HashMap<String, HashMap<String, String>> incrTimestampMap) {
+ this.incrTimeRanges = incrTimestampMap;
+ }
+
+ /**
+ * Get the incremental time range string in the format of:
+ * t1,rs1:ts,rs2:ts,...;t2,rs1:ts,rs2:ts,...;t3,rs1:ts,rs2:ts,...
+ */
+ private String getIncrTimestampStr() {
+ StringBuilder sb = new StringBuilder();
+ for (Entry<String, HashMap<String, String>> tableEntry : this.getIncrTimestamps().entrySet()) {
+ sb.append(tableEntry.getKey() + ","); // table
+ for (Entry<String, String> rsEntry : tableEntry.getValue().entrySet()) {
+ sb.append(rsEntry.getKey() + ":"); // region server
+ sb.append(rsEntry.getValue() + ","); // timestamp
+ }
+ if (sb.length() > 1 && sb.charAt(sb.length() - 1) == ',') {
+ sb.deleteCharAt(sb.length() - 1);
+ }
+ sb.append(";");
+ }
+ if (sb.length() > 1 && sb.charAt(sb.length() - 1) == ';') {
+ sb.deleteCharAt(sb.length() - 1);
+ }
+ return sb.toString();
+ }
+
+ public Map<String, HashMap<String, String>> getIncrTimestamps() {
+ if (this.incrTimeRanges == null) {
+ this.incrTimeRanges = new HashMap<String, HashMap<String, String>>();
+ }
+ return this.incrTimeRanges;
+ }
+
+ /**
+ * Load incremental timestamps from a given string, and store them in the collection. The
+ * timestamps in string is in the format of
+ * t1,rs1:ts,rs2:ts,...;t2,rs1:ts,rs2:ts,...;t3,rs1:ts,rs2:ts,...
+ * @param timeRangesInStr Incremental time ranges in string
+ */
+ private void loadIncrementalTimeRanges(String timeRangesStr) throws IOException {
+
+ LOG.debug("Loading table's incremental time ranges of region servers from string in manifest: "
+ + timeRangesStr);
+
+ Map<String, HashMap<String, String>> timeRangeMap = this.getIncrTimestamps();
+
+ String[] entriesOfTables = timeRangesStr.split(";");
+ for (int i = 0; i < entriesOfTables.length; i++) {
+ String[] itemsForTable = entriesOfTables[i].split(",");
+
+ // validate the incremental timestamps string format for a table:
+ // t1,rs1:ts,rs2:ts,...
+ if (itemsForTable.length < 1) {
+ String errorMsg = "Wrong incremental time range format: " + timeRangesStr;
+ LOG.error(errorMsg);
+ throw new IOException(errorMsg);
+ }
+
+ HashMap<String, String> rsTimestampMap = new HashMap<String, String>();
+ for (int j = 1; j < itemsForTable.length; j++) {
+ String[] rsTsEntry = itemsForTable[j].split(":");
+
+ // validate the incremental timestamps string format for a region server:
+ // rs1:ts
+ if (rsTsEntry.length != 2) {
+ String errorMsg = "Wrong incremental timestamp format: " + itemsForTable[j];
+ LOG.error(errorMsg);
+ throw new IOException(errorMsg);
+ }
+
+ // an entry for timestamp of a region server
+ rsTimestampMap.put(rsTsEntry[0], rsTsEntry[1]);
+ }
+
+ timeRangeMap.put(itemsForTable[0], rsTimestampMap);
+ }
+
+ // all entries have been loaded
+ LOG.debug(entriesOfTables.length + " tables' incremental time ranges have been loaded.");
+ }
+
+ /**
+ * Get the image list of this backup for restore in time order.
+ * @param reverse If true, then output in reverse order, otherwise in time order from old to new
+ * @return the backup image list for restore in time order
+ */
+ public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
+ TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>();
+ for (BackupImage image : this.getDependency().values()) {
+ restoreImages.put(Long.valueOf(image.startTs), image);
+ }
+ return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values())
+ : (restoreImages.values()));
+ }
+
+ /**
+ * Get the dependent image list for a specific table of this backup in time order from old to new
+ * if want to restore to this backup image level.
+ * @param table table
+ * @return the backup image list for a table in time order
+ */
+ public ArrayList<BackupImage> getDependentListByTable(String table) {
+ ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
+ ArrayList<BackupImage> imageList = getRestoreDependentList(true);
+ for (BackupImage image : imageList) {
+ if (image.hasTable(table)) {
+ tableImageList.add(image);
+ if (image.getType().equals(BackupRestoreConstants.BACKUP_TYPE_FULL)) {
+ break;
+ }
+ }
+ }
+ Collections.reverse(tableImageList);
+ return tableImageList;
+ }
+
+ /**
+ * Get the full dependent image list in the whole dependency scope for a specific table of this
+ * backup in time order from old to new.
+ * @param table table
+ * @return the full backup image list for a table in time order in the whole scope of the
+ * dependency of this image
+ */
+ public ArrayList<BackupImage> getAllDependentListByTable(String table) {
+ ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
+ ArrayList<BackupImage> imageList = getRestoreDependentList(false);
+ for (BackupImage image : imageList) {
+ if (image.hasTable(table)) {
+ tableImageList.add(image);
+ }
+ }
+ return tableImageList;
+ }
+
+ /**
+ * Load dependency from a dependency json string.
+ * @param dependencyStr The dependency string
+ * @throws IOException exception
+ */
+ private void loadDependency(String dependencyStr) throws IOException {
+
+ LOG.debug("Loading dependency: " + dependencyStr);
+
+ String msg = "Dependency is broken in the manifest.";
+ if (dependencyStr.equals(ERROR_DEPENDENCY)) {
+ throw new IOException(msg);
+ }
+
+ ObjectMapper mapper = new ObjectMapper();
+ BackupImage image = null;
+ try {
+ image = mapper.readValue(dependencyStr, BackupImage.class);
+ } catch (JsonParseException e) {
+ LOG.error(msg);
+ throw new IOException(e.getMessage());
+ } catch (JsonMappingException e) {
+ LOG.error(msg);
+ throw new IOException(e.getMessage());
+ } catch (IOException e) {
+ LOG.error(msg);
+ throw new IOException(e.getMessage());
+ }
+ LOG.debug("Manifest's current backup image information:");
+ LOG.debug(" Token: " + image.getBackupId());
+ LOG.debug(" Backup directory: " + image.getRootDir());
+ this.setDependencyMap(this.getDependency(), image);
+
+ LOG.debug("Dependent images map:");
+ for (Entry<String, BackupImage> entry : this.getDependency().entrySet()) {
+ LOG.debug(" " + entry.getKey() + " : " + entry.getValue().getBackupId() + " -- "
+ + entry.getValue().getRootDir());
+ }
+
+ LOG.debug("Dependency has been loaded.");
+ }
+
+ /**
+ * Recursively set the dependency map of the backup images.
+ * @param map The dependency map
+ * @param image The backup image
+ */
+ private void setDependencyMap(Map<String, BackupImage> map, BackupImage image) {
+ if (image == null) {
+ return;
+ } else {
+ map.put(image.getBackupId(), image);
+ for (BackupImage img : image.getAncestors()) {
+ setDependencyMap(map, img);
+ }
+ }
+ }
+
+ /**
+ * Check whether backup image1 could cover backup image2 or not.
+ * @param image1 backup image 1
+ * @param image2 backup image 2
+ * @return true if image1 can cover image2, otherwise false
+ */
+ public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
+ // image1 can cover image2 only when the following conditions are satisfied:
+ // - image1 must not be an incremental image;
+ // - image1 must be taken after image2 has been taken;
+ // - table set of image1 must cover the table set of image2.
+ if (image1.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ return false;
+ }
+ if (image1.getStartTs() < image2.getStartTs()) {
+ return false;
+ }
+ String[] image1TableSet = image1.getTableSet().split(";");
+ String[] image2TableSet = image2.getTableSet().split(";");
+ boolean found = false;
+ for (int i = 0; i < image2TableSet.length; i++) {
+ found = false;
+ for (int j = 0; j < image1TableSet.length; j++) {
+ if (image2TableSet[i].equals(image1TableSet[j])) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ return false;
+ }
+ }
+
+ LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
+ return true;
+ }
+
+ /**
+ * Check whether backup image set could cover a backup image or not.
+ * @param fullImages The backup image set
+ * @param image The target backup image
+ * @return true if fullImages can cover image, otherwise false
+ */
+ public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
+ // fullImages can cover image only when the following conditions are satisfied:
+ // - each image of fullImages must not be an incremental image;
+ // - each image of fullImages must be taken after image has been taken;
+ // - sum table set of fullImages must cover the table set of image.
+ for (BackupImage image1 : fullImages) {
+ if (image1.getType().equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ return false;
+ }
+ if (image1.getStartTs() < image.getStartTs()) {
+ return false;
+ }
+ }
+
+ ArrayList<String> image1TableSet = new ArrayList<String>();
+ for (BackupImage image1 : fullImages) {
+ String[] tableSet = image1.getTableSet().split(";");
+ for (String table : tableSet) {
+ image1TableSet.add(table);
+ }
+ }
+ ArrayList<String> image2TableSet = new ArrayList<String>();
+ String[] tableSet = image.getTableSet().split(";");
+ for (String table : tableSet) {
+ image2TableSet.add(table);
+ }
+
+ for (int i = 0; i < image2TableSet.size(); i++) {
+ if (image1TableSet.contains(image2TableSet.get(i)) == false) {
+ return false;
+ }
+ }
+
+ LOG.debug("Full image set can cover image " + image.getBackupId());
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
new file mode 100644
index 0000000..9610b27
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * HConstants holds a bunch of HBase Backup and Restore constants
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public final class BackupRestoreConstants {
+
+ // constants for znode data keys in backup znode
+ public static final String BACKUP_PROGRESS = "progress";
+ public static final String BACKUP_START_TIME = "startTs";
+ public static final String BACKUP_INPROGRESS_PHASE = "phase";
+ public static final String BACKUP_COMPLETE_TIME = "completeTs";
+ public static final String BACKUP_FAIL_TIME = "failedTs";
+ public static final String BACKUP_FAIL_PHASE = "failedphase";
+ public static final String BACKUP_FAIL_MSG = "failedmessage";
+ public static final String BACKUP_ROOT_PATH = "targetRootDir";
+ public static final String BACKUP_REQUEST_TABLE_LIST = "tablelist";
+ public static final String BACKUP_REQUEST_TYPE = "type";
+ public static final String BACKUP_BYTES_COPIED = "bytescopied";
+ public static final String BACKUP_ANCESTORS = "ancestors";
+ public static final String BACKUP_EXISTINGSNAPSHOT = "snapshot";
+
+ public static final String BACKUP_TYPE_FULL = "full";
+ public static final String BACKUP_TYPE_INCR = "incremental";
+
+ // delimiter in tablename list in restore command
+ public static final String TABLENAME_DELIMITER_IN_COMMAND = ",";
+
+ // delimiter in znode data
+ public static final String ZNODE_DATA_DELIMITER = ",";
+
+ public static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
+
+ public static final String BACKUPID_PREFIX = "backup_";
+
+ public static enum BACKUP_COMMAND {
+ CREATE, CANCEL, DELETE, DESCRIBE, HISTORY, STATUS, CONVERT, MERGE, STOP, SHOW, HELP,
+ }
+
+ private BackupRestoreConstants() {
+ // Can't be instantiated with this ctor.
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java
new file mode 100644
index 0000000..1c38cf4
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreServiceFactory.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyService;
+import org.apache.hadoop.hbase.backup.mapreduce.MapReduceRestoreService;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.util.ReflectionUtils;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class BackupRestoreServiceFactory {
+
+ public final static String HBASE_INCR_RESTORE_IMPL_CLASS = "hbase.incremental.restore.class";
+ public final static String HBASE_BACKUP_COPY_IMPL_CLASS = "hbase.backup.copy.class";
+
+ private BackupRestoreServiceFactory(){
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ /**
+ * Gets incremental restore service
+ * @param conf - configuration
+ * @return incremental backup service instance
+ */
+ public static IncrementalRestoreService getIncrementalRestoreService(Configuration conf) {
+ Class<? extends IncrementalRestoreService> cls =
+ conf.getClass(HBASE_INCR_RESTORE_IMPL_CLASS, MapReduceRestoreService.class,
+ IncrementalRestoreService.class);
+ return ReflectionUtils.newInstance(cls, conf);
+ }
+
+ /**
+ * Gets backup copy service
+ * @param conf - configuration
+ * @return backup copy service
+ */
+ public static BackupCopyService getBackupCopyService(Configuration conf) {
+ Class<? extends BackupCopyService> cls =
+ conf.getClass(HBASE_BACKUP_COPY_IMPL_CLASS, MapReduceBackupCopyService.class,
+ BackupCopyService.class);
+ return ReflectionUtils.newInstance(cls, conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
new file mode 100644
index 0000000..c26e1d4
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Backup status and related information encapsulated for a table.
+ * At this moment only TargetDir and SnapshotName is encapsulated here.
+ * future Jira will be implemented for progress, bytesCopies, phase, etc.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupStatus implements Serializable {
+
+ private static final long serialVersionUID = -5968397963548535982L;
+
+ // table name for backup
+ private String table;
+
+ // target directory of the backup image for this table
+ private String targetDir;
+
+ // snapshot name for offline/online snapshot
+ private String snapshotName = null;
+
+ public BackupStatus(String table, String targetRootDir, String backupId) {
+ this.table = table;
+ this.targetDir = HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
+ }
+
+ public String getSnapshotName() {
+ return snapshotName;
+ }
+
+ public void setSnapshotName(String snapshotName) {
+ this.snapshotName = snapshotName;
+ }
+
+ public String getTargetDir() {
+ return targetDir;
+ }
+
+ public String getTable() {
+ return table;
+ }
+}
[16/29] hbase git commit: HBASE-14796 Enhance the Gets in the
connector (Zhan Zhang)
Posted by sy...@apache.org.
HBASE-14796 Enhance the Gets in the connector (Zhan Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6868c636
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6868c636
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6868c636
Branch: refs/heads/hbase-12439
Commit: 6868c6366002d5b4e25980f37ede8839e7a7e92d
Parents: 2fba25b
Author: tedyu <yu...@gmail.com>
Authored: Mon Dec 28 15:48:10 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Mon Dec 28 15:48:10 2015 -0800
----------------------------------------------------------------------
.../hadoop/hbase/spark/DefaultSource.scala | 33 ++-----
.../hadoop/hbase/spark/datasources/Bound.scala | 24 +++++
.../spark/datasources/HBaseResources.scala | 14 +++
.../spark/datasources/HBaseSparkConf.scala | 2 +
.../spark/datasources/HBaseTableScanRDD.scala | 92 +++++++++++++++++---
5 files changed, 127 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6868c636/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala
index 73cab3c..b6d7982 100644
--- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala
+++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala
@@ -159,6 +159,10 @@ case class HBaseRelation (val tableName:String,
.getOrElse(sqlContext.sparkContext.getConf.getInt(
HBaseSparkConf.BATCH_NUM, HBaseSparkConf.defaultBatchNum))
+ val bulkGetSize = parameters.get(HBaseSparkConf.BULKGET_SIZE).map(_.toInt)
+ .getOrElse(sqlContext.sparkContext.getConf.getInt(
+ HBaseSparkConf.BULKGET_SIZE, HBaseSparkConf.defaultBulkGetSize))
+
//create or get latest HBaseContext
@transient val hbaseContext:HBaseContext = if (useHBaseContext) {
LatestHBaseContextCache.latest
@@ -267,6 +271,7 @@ case class HBaseRelation (val tableName:String,
None
}
val hRdd = new HBaseTableScanRDD(this, pushDownFilterJava, requiredQualifierDefinitionList.seq)
+ pushDownRowKeyFilter.points.foreach(hRdd.addPoint(_))
pushDownRowKeyFilter.ranges.foreach(hRdd.addRange(_))
var resultRDD: RDD[Row] = {
val tmp = hRdd.map{ r =>
@@ -280,34 +285,6 @@ case class HBaseRelation (val tableName:String,
}
}
- //If there are gets then we can get them from the driver and union that rdd in
- // with the rest of the values.
- if (getList.size() > 0) {
- val connection =
- ConnectionFactory.createConnection(hbaseContext.tmpHdfsConfiguration)
- try {
- val table = connection.getTable(TableName.valueOf(tableName))
- try {
- val results = table.get(getList)
- val rowList = mutable.MutableList[Row]()
- for (i <- 0 until results.length) {
- val rowArray = requiredColumns.map(c =>
- DefaultSourceStaticUtils.getValue(c, schemaMappingDefinition, results(i)))
- rowList += Row.fromSeq(rowArray)
- }
- val getRDD = sqlContext.sparkContext.parallelize(rowList)
- if (resultRDD == null) resultRDD = getRDD
- else {
- resultRDD = resultRDD.union(getRDD)
- }
- } finally {
- table.close()
- }
- } finally {
- connection.close()
- }
- }
-
if (resultRDD == null) {
val scan = new Scan()
scan.setCacheBlocks(blockCacheEnable)
http://git-wip-us.apache.org/repos/asf/hbase/blob/6868c636/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Bound.scala
----------------------------------------------------------------------
diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Bound.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Bound.scala
index 0f6098d..8e03e95 100644
--- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Bound.scala
+++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Bound.scala
@@ -87,3 +87,27 @@ object Ranges {
}
}
+object Points {
+ def and(r: Range, ps: Seq[Array[Byte]]): Seq[Array[Byte]] = {
+ ps.flatMap { p =>
+ if (ord.compare(r.lower.get.b, p) <= 0) {
+ // if region lower bound is less or equal to the point
+ if (r.upper.isDefined) {
+ // if region upper bound is defined
+ if (ord.compare(r.upper.get.b, p) > 0) {
+ // if the upper bound is greater than the point (because upper bound is exclusive)
+ Some(p)
+ } else {
+ None
+ }
+ } else {
+ // if the region upper bound is not defined (infinity)
+ Some(p)
+ }
+ } else {
+ None
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/6868c636/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseResources.scala
----------------------------------------------------------------------
diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseResources.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseResources.scala
index 19a6ea7..14c5fd0 100644
--- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseResources.scala
+++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseResources.scala
@@ -38,6 +38,12 @@ case class ScanResource(tbr: TableResource, rs: ResultScanner) extends Resource
}
}
+case class GetResource(tbr: TableResource, rs: Array[Result]) extends Resource {
+ def release() {
+ tbr.release()
+ }
+}
+
trait ReferencedResource {
var count: Int = 0
def init(): Unit
@@ -100,6 +106,10 @@ case class TableResource(relation: HBaseRelation) extends ReferencedResource {
def getScanner(scan: Scan): ScanResource = releaseOnException {
ScanResource(this, table.getScanner(scan))
}
+
+ def get(list: java.util.List[org.apache.hadoop.hbase.client.Get]) = releaseOnException {
+ GetResource(this, table.get(list))
+ }
}
case class RegionResource(relation: HBaseRelation) extends ReferencedResource {
@@ -138,6 +148,10 @@ object HBaseResources{
sr.rs
}
+ implicit def GetResToResult(gr: GetResource): Array[Result] = {
+ gr.rs
+ }
+
implicit def TableResToTable(tr: TableResource): Table = {
tr.table
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6868c636/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseSparkConf.scala
----------------------------------------------------------------------
diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseSparkConf.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseSparkConf.scala
index 67580b0..5e11356 100644
--- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseSparkConf.scala
+++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseSparkConf.scala
@@ -29,4 +29,6 @@ object HBaseSparkConf{
val defaultCachingSize = 1000
val BATCH_NUM = "spark.hbase.batchNum"
val defaultBatchNum = 1000
+ val BULKGET_SIZE = "spark.hbase.bulkGetSize"
+ val defaultBulkGetSize = 1000
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6868c636/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala
----------------------------------------------------------------------
diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala
index eb9d39a..f288c34 100644
--- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala
+++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala
@@ -17,6 +17,8 @@
package org.apache.hadoop.hbase.spark.datasources
+import java.util.ArrayList
+
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.spark.{ScanRange, SchemaQualifierDefinition, HBaseRelation, SparkSQLPushDownFilter}
import org.apache.hadoop.hbase.spark.hbase._
@@ -32,7 +34,12 @@ class HBaseTableScanRDD(relation: HBaseRelation,
val columns: Seq[SchemaQualifierDefinition] = Seq.empty
)extends RDD[Result](relation.sqlContext.sparkContext, Nil) with Logging {
private def sparkConf = SparkEnv.get.conf
- var ranges = Seq.empty[Range]
+ @transient var ranges = Seq.empty[Range]
+ @transient var points = Seq.empty[Array[Byte]]
+ def addPoint(p: Array[Byte]) {
+ points :+= p
+ }
+
def addRange(r: ScanRange) = {
val lower = if (r.lowerBound != null && r.lowerBound.length > 0) {
Some(Bound(r.lowerBound, r.isLowerBoundEqualTo))
@@ -65,12 +72,13 @@ class HBaseTableScanRDD(relation: HBaseRelation,
logDebug(s"There are ${regions.size} regions")
val ps = regions.flatMap { x =>
val rs = Ranges.and(Range(x), ranges)
- if (rs.size > 0) {
+ val ps = Points.and(Range(x), points)
+ if (rs.size > 0 || ps.size > 0) {
if(log.isDebugEnabled) {
rs.foreach(x => logDebug(x.toString))
}
idx += 1
- Some(HBaseScanPartition(idx - 1, x, rs, SerializedFilter.toSerializedTypedFilter(filter)))
+ Some(HBaseScanPartition(idx - 1, x, rs, ps, SerializedFilter.toSerializedTypedFilter(filter)))
} else {
None
}
@@ -86,6 +94,57 @@ class HBaseTableScanRDD(relation: HBaseRelation,
}.toSeq
}
+ private def buildGets(
+ tbr: TableResource,
+ g: Seq[Array[Byte]],
+ filter: Option[SparkSQLPushDownFilter],
+ columns: Seq[SchemaQualifierDefinition]): Iterator[Result] = {
+ g.grouped(relation.bulkGetSize).flatMap{ x =>
+ val gets = new ArrayList[Get]()
+ x.foreach{ y =>
+ val g = new Get(y)
+ columns.foreach { d =>
+ if (d.columnFamilyBytes.length > 0) {
+ g.addColumn(d.columnFamilyBytes, d.qualifierBytes)
+ }
+ }
+ filter.foreach(g.setFilter(_))
+ gets.add(g)
+ }
+ val tmp = tbr.get(gets)
+ rddResources.addResource(tmp)
+ toResultIterator(tmp)
+ }
+ }
+
+ private def toResultIterator(result: GetResource): Iterator[Result] = {
+ val iterator = new Iterator[Result] {
+ var idx = 0
+ var cur: Option[Result] = None
+ override def hasNext: Boolean = {
+ while(idx < result.length && cur.isEmpty) {
+ val r = result(idx)
+ idx += 1
+ if (!r.isEmpty) {
+ cur = Some(r)
+ }
+ }
+ if (cur.isEmpty) {
+ rddResources.release(result)
+ }
+ cur.isDefined
+ }
+ override def next(): Result = {
+ hasNext
+ val ret = cur.get
+ cur = None
+ ret
+ }
+ }
+ iterator
+ }
+
+
private def buildScan(range: Range,
filter: Option[SparkSQLPushDownFilter],
columns: Seq[SchemaQualifierDefinition]): Scan = {
@@ -130,6 +189,7 @@ class HBaseTableScanRDD(relation: HBaseRelation,
}
iterator
}
+
lazy val rddResources = RDDResources(new mutable.HashSet[Resource]())
private def close() {
@@ -138,18 +198,29 @@ class HBaseTableScanRDD(relation: HBaseRelation,
override def compute(split: Partition, context: TaskContext): Iterator[Result] = {
val partition = split.asInstanceOf[HBaseScanPartition]
-
+ val filter = SerializedFilter.fromSerializedFilter(partition.sf)
val scans = partition.scanRanges
- .map(buildScan(_, SerializedFilter.fromSerializedFilter(partition.sf), columns))
+ .map(buildScan(_, filter, columns))
val tableResource = TableResource(relation)
context.addTaskCompletionListener(context => close())
- val sIts = scans.par
- .map(tableResource.getScanner(_))
- .map(toResultIterator(_))
+ val points = partition.points
+ val gIt: Iterator[Result] = {
+ if (points.isEmpty) {
+ Iterator.empty: Iterator[Result]
+ } else {
+ buildGets(tableResource, points, filter, columns)
+ }
+ }
+ val rIts = scans.par
+ .map { scan =>
+ val scanner = tableResource.getScanner(scan)
+ rddResources.addResource(scanner)
+ scanner
+ }.map(toResultIterator(_))
.fold(Iterator.empty: Iterator[Result]){ case (x, y) =>
x ++ y
- }
- sIts
+ } ++ gIt
+ rIts
}
}
@@ -176,6 +247,7 @@ private[hbase] case class HBaseScanPartition(
override val index: Int,
val regions: HBaseRegion,
val scanRanges: Seq[Range],
+ val points: Seq[Array[Byte]],
val sf: SerializedFilter) extends Partition
case class RDDResources(set: mutable.HashSet[Resource]) {
[22/29] hbase git commit: HBASE-15050 Block Ref counting does not
work in Region Split cases (Ram)
Posted by sy...@apache.org.
HBASE-15050 Block Ref counting does not work in Region Split cases (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/07b62367
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/07b62367
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/07b62367
Branch: refs/heads/hbase-12439
Commit: 07b623670647686084f8f5fd2038e2bafcfdac54
Parents: 413d663
Author: ramkrishna <ra...@gmail.com>
Authored: Wed Dec 30 14:39:03 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Wed Dec 30 14:39:03 2015 +0530
----------------------------------------------------------------------
.../hadoop/hbase/io/HalfStoreFileReader.java | 8 +++
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 8 ++-
.../client/TestBlockEvictionFromClient.java | 53 ++++++++++++++++++++
3 files changed, 68 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/07b62367/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index 0dd7742..067d24c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -314,6 +314,10 @@ public class HalfStoreFileReader extends StoreFile.Reader {
}
} catch (IOException e) {
LOG.warn("Failed seekBefore " + Bytes.toStringBinary(this.splitkey), e);
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
}
return null;
}
@@ -335,6 +339,10 @@ public class HalfStoreFileReader extends StoreFile.Reader {
firstKeySeeked = true;
} catch (IOException e) {
LOG.warn("Failed seekTo first KV in the file", e);
+ } finally {
+ if(scanner != null) {
+ scanner.close();
+ }
}
}
return this.firstKey;
http://git-wip-us.apache.org/repos/asf/hbase/blob/07b62367/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
index 4e2ca7d..fcf7b5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
@@ -986,7 +986,13 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
return new KeyValue.KeyOnlyKeyValue(keyBuf.array(), keyBuf.arrayOffset()
+ keyPair.getSecond(), currKeyLen);
} else {
- return new ByteBufferedKeyOnlyKeyValue(keyBuf, keyPair.getSecond(), currKeyLen);
+ // Better to do a copy here instead of holding on to this BB so that
+ // we could release the blocks referring to this key. This key is specifically used
+ // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner
+ // every time. So holding onto the BB (incase of DBB) is not advised here.
+ byte[] key = new byte[currKeyLen];
+ ByteBufferUtils.copyFromBufferToArray(key, keyBuf, keyPair.getSecond(), 0, currKeyLen);
+ return new KeyValue.KeyOnlyKeyValue(key, 0, currKeyLen);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/07b62367/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
index a812623..f4d668c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
@@ -74,6 +74,8 @@ public class TestBlockEvictionFromClient {
private static int NO_OF_THREADS = 3;
private static byte[] ROW = Bytes.toBytes("testRow");
private static byte[] ROW1 = Bytes.toBytes("testRow1");
+ private static byte[] ROW2 = Bytes.toBytes("testRow2");
+ private static byte[] ROW3 = Bytes.toBytes("testRow3");
private static byte[] FAMILY = Bytes.toBytes("testFamily");
private static byte[][] FAMILIES_1 = new byte[1][0];
private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
@@ -554,6 +556,57 @@ public class TestBlockEvictionFromClient {
}
@Test
+ public void testBlockRefCountAfterSplits() throws IOException, InterruptedException {
+ HTable table = null;
+ try {
+ TableName tableName = TableName.valueOf("testBlockRefCountAfterSplits");
+ table = TEST_UTIL.createTable(tableName, FAMILIES_1, 1, 1024);
+ // get the block cache and region
+ RegionLocator locator = table.getRegionLocator();
+ String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
+ Region region =
+ TEST_UTIL.getRSForFirstRegionInTable(tableName).getFromOnlineRegions(regionName);
+ Store store = region.getStores().iterator().next();
+ CacheConfig cacheConf = store.getCacheConfig();
+ cacheConf.setEvictOnClose(true);
+ BlockCache cache = cacheConf.getBlockCache();
+
+ Put put = new Put(ROW);
+ put.addColumn(FAMILY, QUALIFIER, data);
+ table.put(put);
+ region.flush(true);
+ put = new Put(ROW1);
+ put.addColumn(FAMILY, QUALIFIER, data);
+ table.put(put);
+ region.flush(true);
+ byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
+ put = new Put(ROW2);
+ put.addColumn(FAMILY, QUALIFIER2, data2);
+ table.put(put);
+ put = new Put(ROW3);
+ put.addColumn(FAMILY, QUALIFIER2, data2);
+ table.put(put);
+ region.flush(true);
+ TEST_UTIL.getAdmin().split(tableName, ROW1);
+ List<HRegionInfo> tableRegions = TEST_UTIL.getAdmin().getTableRegions(tableName);
+ // Wait for splits
+ while (tableRegions.size() != 2) {
+ tableRegions = TEST_UTIL.getAdmin().getTableRegions(tableName);
+ Thread.sleep(100);
+ }
+ region.compact(true);
+ Iterator<CachedBlock> iterator = cache.iterator();
+ // Though the split had created the HalfStorefileReader - the firstkey and lastkey scanners
+ // should be closed inorder to return those blocks
+ iterateBlockCache(cache, iterator);
+ } finally {
+ if (table != null) {
+ table.close();
+ }
+ }
+ }
+
+ @Test
public void testMultiGets() throws IOException, InterruptedException {
HTable table = null;
try {
[06/29] hbase git commit: HBASE-14030 HBase Backup/Restore Phase 1
(Vladimir Rodionov)
Posted by sy...@apache.org.
HBASE-14030 HBase Backup/Restore Phase 1 (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de69f0df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de69f0df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de69f0df
Branch: refs/heads/hbase-12439
Commit: de69f0df34a6cb00a9fd2e370fdf898274871294
Parents: 5eefe13
Author: tedyu <yu...@gmail.com>
Authored: Sun Dec 27 10:02:09 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Sun Dec 27 10:02:09 2015 -0800
----------------------------------------------------------------------
bin/hbase | 6 +
.../org/apache/hadoop/hbase/HConstants.java | 445 +++++-----
hbase-server/pom.xml | 10 +
.../hadoop/hbase/backup/BackupClient.java | 338 ++++++++
.../hadoop/hbase/backup/BackupCommands.java | 147 ++++
.../hadoop/hbase/backup/BackupContext.java | 318 ++++++++
.../hadoop/hbase/backup/BackupCopyService.java | 37 +
.../hadoop/hbase/backup/BackupException.java | 85 ++
.../hadoop/hbase/backup/BackupHandler.java | 744 +++++++++++++++++
.../hadoop/hbase/backup/BackupManager.java | 488 +++++++++++
.../hadoop/hbase/backup/BackupManifest.java | 814 +++++++++++++++++++
.../hbase/backup/BackupRestoreConstants.java | 66 ++
.../backup/BackupRestoreServiceFactory.java | 61 ++
.../hadoop/hbase/backup/BackupStatus.java | 67 ++
.../hadoop/hbase/backup/BackupSystemTable.java | 642 +++++++++++++++
.../hbase/backup/BackupSystemTableHelper.java | 314 +++++++
.../apache/hadoop/hbase/backup/BackupUtil.java | 564 +++++++++++++
.../hadoop/hbase/backup/HBackupFileSystem.java | 511 ++++++++++++
.../hbase/backup/IncrementalBackupManager.java | 269 ++++++
.../hbase/backup/IncrementalRestoreService.java | 33 +
.../hadoop/hbase/backup/RestoreClient.java | 496 +++++++++++
.../apache/hadoop/hbase/backup/RestoreUtil.java | 503 ++++++++++++
.../mapreduce/MapReduceBackupCopyService.java | 292 +++++++
.../mapreduce/MapReduceRestoreService.java | 72 ++
.../hbase/backup/master/BackupLogCleaner.java | 121 +++
.../master/LogRollMasterProcedureManager.java | 129 +++
.../regionserver/LogRollBackupSubprocedure.java | 138 ++++
.../LogRollBackupSubprocedurePool.java | 137 ++++
.../LogRollRegionServerProcedureManager.java | 168 ++++
.../BaseCoordinatedStateManager.java | 17 +
.../coordination/ZkCoordinatedStateManager.java | 20 +-
.../hadoop/hbase/mapreduce/WALPlayer.java | 54 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +
.../procedure/RegionServerProcedureManager.java | 2 +-
.../RegionServerProcedureManagerHost.java | 3 +-
.../procedure/ZKProcedureCoordinatorRpcs.java | 2 +-
.../hbase/procedure/ZKProcedureMemberRpcs.java | 83 +-
.../RegionServerFlushTableProcedureManager.java | 2 +-
.../hbase/regionserver/HRegionServer.java | 4 +-
.../snapshot/RegionServerSnapshotManager.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 67 +-
.../hadoop/hbase/snapshot/SnapshotCopy.java | 42 +
.../hadoop/hbase/wal/DefaultWALProvider.java | 7 +-
.../hadoop/hbase/backup/TestBackupBase.java | 194 +++++
.../hbase/backup/TestBackupBoundaryTests.java | 99 +++
.../hbase/backup/TestBackupLogCleaner.java | 161 ++++
.../hbase/backup/TestBackupSystemTable.java | 341 ++++++++
.../hadoop/hbase/backup/TestFullBackup.java | 83 ++
.../hadoop/hbase/backup/TestFullRestore.java | 166 ++++
.../hbase/backup/TestIncrementalBackup.java | 179 ++++
.../hadoop/hbase/backup/TestRemoteBackup.java | 44 +
.../hadoop/hbase/backup/TestRemoteRestore.java | 54 ++
.../hbase/backup/TestRestoreBoundaryTests.java | 85 ++
.../procedure/SimpleRSProcedureManager.java | 2 +-
54 files changed, 9389 insertions(+), 341 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 5064451..9fd903a 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -99,6 +99,8 @@ if [ $# = 0 ]; then
echo " pe Run PerformanceEvaluation"
echo " ltt Run LoadTestTool"
echo " version Print the version"
+ echo " backup backup tables for recovery"
+ echo " restore restore tables from existing backup image"
echo " CLASSNAME Run the class named CLASSNAME"
exit 1
fi
@@ -303,6 +305,10 @@ elif [ "$COMMAND" = "hfile" ] ; then
CLASS='org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter'
elif [ "$COMMAND" = "zkcli" ] ; then
CLASS="org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServer"
+elif [ "$COMMAND" = "backup" ] ; then
+ CLASS='org.apache.hadoop.hbase.backup.BackupClient'
+elif [ "$COMMAND" = "restore" ] ; then
+ CLASS='org.apache.hadoop.hbase.backup.RestoreClient'
elif [ "$COMMAND" = "upgrade" ] ; then
echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0."
echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading."
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 6fafad3..0f572e6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -40,11 +40,11 @@ import org.apache.hadoop.hbase.util.Bytes;
public final class HConstants {
// NOTICE!!!! Please do not add a constants here, unless they are referenced by a lot of classes.
- //Bytes.UTF8_ENCODING should be updated if this changed
+ // Bytes.UTF8_ENCODING should be updated if this changed
/** When we encode strings, we always specify UTF8 encoding */
public static final String UTF8_ENCODING = "UTF-8";
- //Bytes.UTF8_CHARSET should be updated if this changed
+ // Bytes.UTF8_CHARSET should be updated if this changed
/** When we encode strings, we always specify UTF8 encoding */
public static final Charset UTF8_CHARSET = Charset.forName(UTF8_ENCODING);
/**
@@ -55,9 +55,9 @@ public final class HConstants {
/** Used as a magic return value while optimized index key feature enabled(HBASE-7845) */
public final static int INDEX_KEY_MAGIC = -2;
/*
- * Name of directory that holds recovered edits written by the wal log
- * splitting code, one per region
- */
+ * Name of directory that holds recovered edits written by the wal log splitting code, one per
+ * region
+ */
public static final String RECOVERED_EDITS_DIR = "recovered.edits";
/**
* The first four bytes of Hadoop RPC connections
@@ -70,27 +70,24 @@ public final class HConstants {
/** The size data structures with minor version is 0 */
public static final int HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
+ Bytes.SIZEOF_LONG;
- /** The size of a version 2 HFile block header, minor version 1.
- * There is a 1 byte checksum type, followed by a 4 byte bytesPerChecksum
- * followed by another 4 byte value to store sizeofDataOnDisk.
+ /**
+ * The size of a version 2 HFile block header, minor version 1. There is a 1 byte checksum type,
+ * followed by a 4 byte bytesPerChecksum followed by another 4 byte value to store
+ * sizeofDataOnDisk.
*/
- public static final int HFILEBLOCK_HEADER_SIZE = HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM +
- Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT;
+ public static final int HFILEBLOCK_HEADER_SIZE = HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM
+ + Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT;
/** Just an array of bytes of the right size. */
public static final byte[] HFILEBLOCK_DUMMY_HEADER = new byte[HFILEBLOCK_HEADER_SIZE];
- //End HFileBlockConstants.
+ // End HFileBlockConstants.
/**
* Status codes used for return values of bulk operations.
*/
@InterfaceAudience.Private
public enum OperationStatusCode {
- NOT_RUN,
- SUCCESS,
- BAD_FAMILY,
- SANITY_CHECK_FAILURE,
- FAILURE;
+ NOT_RUN, SUCCESS, BAD_FAMILY, SANITY_CHECK_FAILURE, FAILURE;
}
/** long constant for zero */
@@ -104,19 +101,16 @@ public final class HConstants {
public static final String VERSION_FILE_NAME = "hbase.version";
/**
- * Current version of file system.
- * Version 4 supports only one kind of bloom filter.
- * Version 5 changes versions in catalog table regions.
- * Version 6 enables blockcaching on catalog tables.
- * Version 7 introduces hfile -- hbase 0.19 to 0.20..
- * Version 8 introduces namespace
+ * Current version of file system. Version 4 supports only one kind of bloom filter. Version 5
+ * changes versions in catalog table regions. Version 6 enables blockcaching on catalog tables.
+ * Version 7 introduces hfile -- hbase 0.19 to 0.20.. Version 8 introduces namespace
*/
// public static final String FILE_SYSTEM_VERSION = "6";
public static final String FILE_SYSTEM_VERSION = "8";
// Configuration parameters
- //TODO: Is having HBase homed on port 60k OK?
+ // TODO: Is having HBase homed on port 60k OK?
/** Cluster is in distributed mode or not */
public static final String CLUSTER_DISTRIBUTED = "hbase.cluster.distributed";
@@ -131,12 +125,10 @@ public final class HConstants {
public static final String ENSEMBLE_TABLE_NAME = "hbase:ensemble";
/** Config for pluggable region normalizer */
- public static final String HBASE_MASTER_NORMALIZER_CLASS =
- "hbase.master.normalizer.class";
+ public static final String HBASE_MASTER_NORMALIZER_CLASS = "hbase.master.normalizer.class";
/** Config for enabling/disabling pluggable region normalizer */
- public static final String HBASE_NORMALIZER_ENABLED =
- "hbase.normalizer.enabled";
+ public static final String HBASE_NORMALIZER_ENABLED = "hbase.normalizer.enabled";
/** Cluster is standalone or pseudo-distributed */
public static final boolean CLUSTER_IS_LOCAL = false;
@@ -174,21 +166,18 @@ public final class HConstants {
public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
/** Common prefix of ZooKeeper configuration properties */
- public static final String ZK_CFG_PROPERTY_PREFIX =
- "hbase.zookeeper.property.";
+ public static final String ZK_CFG_PROPERTY_PREFIX = "hbase.zookeeper.property.";
- public static final int ZK_CFG_PROPERTY_PREFIX_LEN =
- ZK_CFG_PROPERTY_PREFIX.length();
+ public static final int ZK_CFG_PROPERTY_PREFIX_LEN = ZK_CFG_PROPERTY_PREFIX.length();
/**
- * The ZK client port key in the ZK properties map. The name reflects the
- * fact that this is not an HBase configuration key.
+ * The ZK client port key in the ZK properties map. The name reflects the fact that this is not an
+ * HBase configuration key.
*/
public static final String CLIENT_PORT_STR = "clientPort";
/** Parameter name for the client port that the zookeeper listens on */
- public static final String ZOOKEEPER_CLIENT_PORT =
- ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR;
+ public static final String ZOOKEEPER_CLIENT_PORT = ZK_CFG_PROPERTY_PREFIX + CLIENT_PORT_STR;
/** Default client port that the zookeeper listens on */
public static final int DEFAULT_ZOOKEPER_CLIENT_PORT = 2181;
@@ -208,19 +197,15 @@ public final class HConstants {
public static final String DEFAULT_ZOOKEEPER_ZNODE_PARENT = "/hbase";
/**
- * Parameter name for the limit on concurrent client-side zookeeper
- * connections
+ * Parameter name for the limit on concurrent client-side zookeeper connections
*/
- public static final String ZOOKEEPER_MAX_CLIENT_CNXNS =
- ZK_CFG_PROPERTY_PREFIX + "maxClientCnxns";
+ public static final String ZOOKEEPER_MAX_CLIENT_CNXNS = ZK_CFG_PROPERTY_PREFIX + "maxClientCnxns";
/** Parameter name for the ZK data directory */
- public static final String ZOOKEEPER_DATA_DIR =
- ZK_CFG_PROPERTY_PREFIX + "dataDir";
+ public static final String ZOOKEEPER_DATA_DIR = ZK_CFG_PROPERTY_PREFIX + "dataDir";
/** Parameter name for the ZK tick time */
- public static final String ZOOKEEPER_TICK_TIME =
- ZK_CFG_PROPERTY_PREFIX + "tickTime";
+ public static final String ZOOKEEPER_TICK_TIME = ZK_CFG_PROPERTY_PREFIX + "tickTime";
/** Default limit on concurrent client-side zookeeper connections */
public static final int DEFAULT_ZOOKEPER_MAX_CLIENT_CNXNS = 300;
@@ -244,21 +229,19 @@ public final class HConstants {
public static final int DEFAULT_REGIONSERVER_INFOPORT = 16030;
/** A configuration key for regionserver info port */
- public static final String REGIONSERVER_INFO_PORT =
- "hbase.regionserver.info.port";
+ public static final String REGIONSERVER_INFO_PORT = "hbase.regionserver.info.port";
/** A flag that enables automatic selection of regionserver info port */
- public static final String REGIONSERVER_INFO_PORT_AUTO =
- REGIONSERVER_INFO_PORT + ".auto";
+ public static final String REGIONSERVER_INFO_PORT_AUTO = REGIONSERVER_INFO_PORT + ".auto";
/** Parameter name for what region server implementation to use. */
- public static final String REGION_SERVER_IMPL= "hbase.regionserver.impl";
+ public static final String REGION_SERVER_IMPL = "hbase.regionserver.impl";
/** Parameter name for what master implementation to use. */
- public static final String MASTER_IMPL= "hbase.master.impl";
+ public static final String MASTER_IMPL = "hbase.master.impl";
/** Parameter name for what hbase client implementation to use. */
- public static final String HBASECLIENT_IMPL= "hbase.hbaseclient.impl";
+ public static final String HBASECLIENT_IMPL = "hbase.hbaseclient.impl";
/** Parameter name for how often threads should wake up */
public static final String THREAD_WAKE_FREQUENCY = "hbase.server.thread.wakefrequency";
@@ -293,7 +276,7 @@ public final class HConstants {
/** Parameter name for HBase client operation timeout, which overrides RPC timeout */
public static final String HBASE_CLIENT_META_OPERATION_TIMEOUT =
- "hbase.client.meta.operation.timeout";
+ "hbase.client.meta.operation.timeout";
/** Default HBase client operation timeout, which is tantamount to a blocking call */
public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = 1200000;
@@ -316,9 +299,8 @@ public final class HConstants {
public static final String MIGRATION_NAME = ".migration";
/**
- * The directory from which co-processor/custom filter jars can be loaded
- * dynamically by the region servers. This value can be overridden by the
- * hbase.dynamic.jars.dir config.
+ * The directory from which co-processor/custom filter jars can be loaded dynamically by the
+ * region servers. This value can be overridden by the hbase.dynamic.jars.dir config.
*/
public static final String LIB_DIR = "lib";
@@ -326,8 +308,7 @@ public final class HConstants {
public static final String HREGION_COMPACTIONDIR_NAME = "compaction.dir";
/** Conf key for the max file size after which we split the region */
- public static final String HREGION_MAX_FILESIZE =
- "hbase.hregion.max.filesize";
+ public static final String HREGION_MAX_FILESIZE = "hbase.hregion.max.filesize";
/** Default maximum file size */
public static final long DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024 * 1024L;
@@ -343,25 +324,24 @@ public final class HConstants {
public static final long TABLE_MAX_ROWSIZE_DEFAULT = 1024 * 1024 * 1024L;
/**
- * The max number of threads used for opening and closing stores or store
- * files in parallel
+ * The max number of threads used for opening and closing stores or store files in parallel
*/
public static final String HSTORE_OPEN_AND_CLOSE_THREADS_MAX =
- "hbase.hstore.open.and.close.threads.max";
+ "hbase.hstore.open.and.close.threads.max";
/**
- * The default number for the max number of threads used for opening and
- * closing stores or store files in parallel
+ * The default number for the max number of threads used for opening and closing stores or store
+ * files in parallel
*/
public static final int DEFAULT_HSTORE_OPEN_AND_CLOSE_THREADS_MAX = 1;
/**
- * Block updates if memstore has hbase.hregion.memstore.block.multiplier
- * times hbase.hregion.memstore.flush.size bytes. Useful preventing
- * runaway memstore during spikes in update traffic.
+ * Block updates if memstore has hbase.hregion.memstore.block.multiplier times
+ * hbase.hregion.memstore.flush.size bytes. Useful preventing runaway memstore during spikes in
+ * update traffic.
*/
public static final String HREGION_MEMSTORE_BLOCK_MULTIPLIER =
- "hbase.hregion.memstore.block.multiplier";
+ "hbase.hregion.memstore.block.multiplier";
/**
* Default value for hbase.hregion.memstore.block.multiplier
@@ -369,14 +349,12 @@ public final class HConstants {
public static final int DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER = 4;
/** Conf key for the memstore size at which we flush the memstore */
- public static final String HREGION_MEMSTORE_FLUSH_SIZE =
- "hbase.hregion.memstore.flush.size";
+ public static final String HREGION_MEMSTORE_FLUSH_SIZE = "hbase.hregion.memstore.flush.size";
public static final String HREGION_EDITS_REPLAY_SKIP_ERRORS =
"hbase.hregion.edits.replay.skip.errors";
- public static final boolean DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS =
- false;
+ public static final boolean DEFAULT_HREGION_EDITS_REPLAY_SKIP_ERRORS = false;
/** Maximum value length, enforced on KeyValue construction */
public static final int MAXIMUM_VALUE_LENGTH = Integer.MAX_VALUE - 1;
@@ -412,12 +390,12 @@ public final class HConstants {
// be the first to be reassigned if the server(s) they are being served by
// should go down.
-
/**
* The hbase:meta table's name.
* @deprecated For upgrades of 0.94 to 0.96
*/
- @Deprecated // for compat from 0.94 -> 0.96.
+ @Deprecated
+ // for compat from 0.94 -> 0.96.
public static final byte[] META_TABLE_NAME = TableName.META_TABLE_NAME.getName();
public static final String BASE_NAMESPACE_DIR = "data";
@@ -425,52 +403,52 @@ public final class HConstants {
/** delimiter used between portions of a region name */
public static final int META_ROW_DELIMITER = ',';
- /** The catalog family as a string*/
+ /** The catalog family as a string */
public static final String CATALOG_FAMILY_STR = "info";
/** The catalog family */
- public static final byte [] CATALOG_FAMILY = Bytes.toBytes(CATALOG_FAMILY_STR);
+ public static final byte[] CATALOG_FAMILY = Bytes.toBytes(CATALOG_FAMILY_STR);
/** The RegionInfo qualifier as a string */
public static final String REGIONINFO_QUALIFIER_STR = "regioninfo";
/** The regioninfo column qualifier */
- public static final byte [] REGIONINFO_QUALIFIER = Bytes.toBytes(REGIONINFO_QUALIFIER_STR);
+ public static final byte[] REGIONINFO_QUALIFIER = Bytes.toBytes(REGIONINFO_QUALIFIER_STR);
/** The server column qualifier */
public static final String SERVER_QUALIFIER_STR = "server";
/** The server column qualifier */
- public static final byte [] SERVER_QUALIFIER = Bytes.toBytes(SERVER_QUALIFIER_STR);
+ public static final byte[] SERVER_QUALIFIER = Bytes.toBytes(SERVER_QUALIFIER_STR);
/** The startcode column qualifier */
public static final String STARTCODE_QUALIFIER_STR = "serverstartcode";
/** The startcode column qualifier */
- public static final byte [] STARTCODE_QUALIFIER = Bytes.toBytes(STARTCODE_QUALIFIER_STR);
+ public static final byte[] STARTCODE_QUALIFIER = Bytes.toBytes(STARTCODE_QUALIFIER_STR);
/** The open seqnum column qualifier */
public static final String SEQNUM_QUALIFIER_STR = "seqnumDuringOpen";
/** The open seqnum column qualifier */
- public static final byte [] SEQNUM_QUALIFIER = Bytes.toBytes(SEQNUM_QUALIFIER_STR);
+ public static final byte[] SEQNUM_QUALIFIER = Bytes.toBytes(SEQNUM_QUALIFIER_STR);
/** The state column qualifier */
public static final String STATE_QUALIFIER_STR = "state";
- public static final byte [] STATE_QUALIFIER = Bytes.toBytes(STATE_QUALIFIER_STR);
+ public static final byte[] STATE_QUALIFIER = Bytes.toBytes(STATE_QUALIFIER_STR);
/**
- * The serverName column qualifier. Its the server where the region is
- * transitioning on, while column server is the server where the region is
- * opened on. They are the same when the region is in state OPEN.
+ * The serverName column qualifier. Its the server where the region is transitioning on, while
+ * column server is the server where the region is opened on. They are the same when the region is
+ * in state OPEN.
*/
public static final String SERVERNAME_QUALIFIER_STR = "sn";
- public static final byte [] SERVERNAME_QUALIFIER = Bytes.toBytes(SERVERNAME_QUALIFIER_STR);
+ public static final byte[] SERVERNAME_QUALIFIER = Bytes.toBytes(SERVERNAME_QUALIFIER_STR);
/** The lower-half split region column qualifier */
- public static final byte [] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
+ public static final byte[] SPLITA_QUALIFIER = Bytes.toBytes("splitA");
/** The upper-half split region column qualifier */
- public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
+ public static final byte[] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
/** The lower-half merge region column qualifier */
public static final byte[] MERGEA_QUALIFIER = Bytes.toBytes("mergeA");
@@ -478,32 +456,28 @@ public final class HConstants {
/** The upper-half merge region column qualifier */
public static final byte[] MERGEB_QUALIFIER = Bytes.toBytes("mergeB");
- /** The catalog family as a string*/
+ /** The catalog family as a string */
public static final String TABLE_FAMILY_STR = "table";
/** The catalog family */
- public static final byte [] TABLE_FAMILY = Bytes.toBytes(TABLE_FAMILY_STR);
+ public static final byte[] TABLE_FAMILY = Bytes.toBytes(TABLE_FAMILY_STR);
/** The serialized table state qualifier */
public static final byte[] TABLE_STATE_QUALIFIER = Bytes.toBytes("state");
-
/**
- * The meta table version column qualifier.
- * We keep current version of the meta table in this column in <code>-ROOT-</code>
- * table: i.e. in the 'info:v' column.
+ * The meta table version column qualifier. We keep current version of the meta table in this
+ * column in <code>-ROOT-</code> table: i.e. in the 'info:v' column.
*/
- public static final byte [] META_VERSION_QUALIFIER = Bytes.toBytes("v");
+ public static final byte[] META_VERSION_QUALIFIER = Bytes.toBytes("v");
/**
- * The current version of the meta table.
- * - pre-hbase 0.92. There is no META_VERSION column in the root table
- * in this case. The meta has HTableDescriptor serialized into the HRegionInfo;
- * - version 0 is 0.92 and 0.94. Meta data has serialized HRegionInfo's using
- * Writable serialization, and HRegionInfo's does not contain HTableDescriptors.
- * - version 1 for 0.96+ keeps HRegionInfo data structures, but changes the
- * byte[] serialization from Writables to Protobuf.
- * See HRegionInfo.VERSION
+ * The current version of the meta table. - pre-hbase 0.92. There is no META_VERSION column in the
+ * root table in this case. The meta has HTableDescriptor serialized into the HRegionInfo; -
+ * version 0 is 0.92 and 0.94. Meta data has serialized HRegionInfo's using Writable
+ * serialization, and HRegionInfo's does not contain HTableDescriptors. - version 1 for 0.96+
+ * keeps HRegionInfo data structures, but changes the byte[] serialization from Writables to
+ * Protobuf. See HRegionInfo.VERSION
*/
public static final short META_VERSION = 1;
@@ -512,25 +486,24 @@ public final class HConstants {
/**
* An empty instance.
*/
- public static final byte [] EMPTY_BYTE_ARRAY = new byte [0];
+ public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(EMPTY_BYTE_ARRAY);
/**
* Used by scanners, etc when they want to start at the beginning of a region
*/
- public static final byte [] EMPTY_START_ROW = EMPTY_BYTE_ARRAY;
+ public static final byte[] EMPTY_START_ROW = EMPTY_BYTE_ARRAY;
/**
* Last row in a table.
*/
- public static final byte [] EMPTY_END_ROW = EMPTY_START_ROW;
+ public static final byte[] EMPTY_END_ROW = EMPTY_START_ROW;
/**
- * Used by scanners and others when they're trying to detect the end of a
- * table
- */
- public static final byte [] LAST_ROW = EMPTY_BYTE_ARRAY;
+ * Used by scanners and others when they're trying to detect the end of a table
+ */
+ public static final byte[] LAST_ROW = EMPTY_BYTE_ARRAY;
/**
* Max length a row can have because of the limitation in TFile.
@@ -538,9 +511,8 @@ public final class HConstants {
public static final int MAX_ROW_LENGTH = Short.MAX_VALUE;
/**
- * Timestamp to use when we want to refer to the latest cell.
- * This is the timestamp sent by clients when no timestamp is specified on
- * commit.
+ * Timestamp to use when we want to refer to the latest cell. This is the timestamp sent by
+ * clients when no timestamp is specified on commit.
*/
public static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
@@ -552,17 +524,12 @@ public final class HConstants {
/**
* LATEST_TIMESTAMP in bytes form
*/
- public static final byte [] LATEST_TIMESTAMP_BYTES = {
+ public static final byte[] LATEST_TIMESTAMP_BYTES = {
// big-endian
- (byte) (LATEST_TIMESTAMP >>> 56),
- (byte) (LATEST_TIMESTAMP >>> 48),
- (byte) (LATEST_TIMESTAMP >>> 40),
- (byte) (LATEST_TIMESTAMP >>> 32),
- (byte) (LATEST_TIMESTAMP >>> 24),
- (byte) (LATEST_TIMESTAMP >>> 16),
- (byte) (LATEST_TIMESTAMP >>> 8),
- (byte) LATEST_TIMESTAMP,
- };
+ (byte) (LATEST_TIMESTAMP >>> 56), (byte) (LATEST_TIMESTAMP >>> 48),
+ (byte) (LATEST_TIMESTAMP >>> 40), (byte) (LATEST_TIMESTAMP >>> 32),
+ (byte) (LATEST_TIMESTAMP >>> 24), (byte) (LATEST_TIMESTAMP >>> 16),
+ (byte) (LATEST_TIMESTAMP >>> 8), (byte) LATEST_TIMESTAMP };
/**
* Define for 'return-all-versions'.
@@ -572,7 +539,7 @@ public final class HConstants {
/**
* Unlimited time-to-live.
*/
-// public static final int FOREVER = -1;
+ // public static final int FOREVER = -1;
public static final int FOREVER = Integer.MAX_VALUE;
/**
@@ -587,10 +554,10 @@ public final class HConstants {
public static final int HOUR_IN_SECONDS = 60 * 60;
public static final int MINUTE_IN_SECONDS = 60;
- //TODO: although the following are referenced widely to format strings for
- // the shell. They really aren't a part of the public API. It would be
- // nice if we could put them somewhere where they did not need to be
- // public. They could have package visibility
+ // TODO: although the following are referenced widely to format strings for
+ // the shell. They really aren't a part of the public API. It would be
+ // nice if we could put them somewhere where they did not need to be
+ // public. They could have package visibility
public static final String NAME = "NAME";
public static final String VERSIONS = "VERSIONS";
public static final String IN_MEMORY = "IN_MEMORY";
@@ -598,45 +565,38 @@ public final class HConstants {
public static final String CONFIGURATION = "CONFIGURATION";
/**
- * Retrying we multiply hbase.client.pause setting by what we have in this array until we
- * run out of array items. Retries beyond this use the last number in the array. So, for
- * example, if hbase.client.pause is 1 second, and maximum retries count
- * hbase.client.retries.number is 10, we will retry at the following intervals:
- * 1, 2, 3, 5, 10, 20, 40, 100, 100, 100.
- * With 100ms, a back-off of 200 means 20s
+ * Retrying we multiply hbase.client.pause setting by what we have in this array until we run out
+ * of array items. Retries beyond this use the last number in the array. So, for example, if
+ * hbase.client.pause is 1 second, and maximum retries count hbase.client.retries.number is 10, we
+ * will retry at the following intervals: 1, 2, 3, 5, 10, 20, 40, 100, 100, 100. With 100ms, a
+ * back-off of 200 means 20s
*/
- public static final int [] RETRY_BACKOFF = {1, 2, 3, 5, 10, 20, 40, 100, 100, 100, 100, 200, 200};
+ public static final int[] RETRY_BACKOFF =
+ { 1, 2, 3, 5, 10, 20, 40, 100, 100, 100, 100, 200, 200 };
public static final String REGION_IMPL = "hbase.hregion.impl";
/** modifyTable op for replacing the table descriptor */
@InterfaceAudience.Private
public static enum Modify {
- CLOSE_REGION,
- TABLE_COMPACT,
- TABLE_FLUSH,
- TABLE_MAJOR_COMPACT,
- TABLE_SET_HTD,
- TABLE_SPLIT
+ CLOSE_REGION, TABLE_COMPACT, TABLE_FLUSH, TABLE_MAJOR_COMPACT, TABLE_SET_HTD, TABLE_SPLIT
}
/**
- * Scope tag for locally scoped data.
- * This data will not be replicated.
+ * Scope tag for locally scoped data. This data will not be replicated.
*/
public static final int REPLICATION_SCOPE_LOCAL = 0;
/**
- * Scope tag for globally scoped data.
- * This data will be replicated to all peers.
+ * Scope tag for globally scoped data. This data will be replicated to all peers.
*/
public static final int REPLICATION_SCOPE_GLOBAL = 1;
/**
- * Default cluster ID, cannot be used to identify a cluster so a key with
- * this value means it wasn't meant for replication.
+ * Default cluster ID, cannot be used to identify a cluster so a key with this value means it
+ * wasn't meant for replication.
*/
- public static final UUID DEFAULT_CLUSTER_ID = new UUID(0L,0L);
+ public static final UUID DEFAULT_CLUSTER_ID = new UUID(0L, 0L);
/**
* Parameter name for maximum number of bytes returned when calling a scanner's next method.
@@ -653,27 +613,22 @@ public final class HConstants {
"hbase.server.scanner.max.result.size";
/**
- * Maximum number of bytes returned when calling a scanner's next method.
- * Note that when a single row is larger than this limit the row is still
- * returned completely.
- *
- * The default value is 2MB.
+ * Maximum number of bytes returned when calling a scanner's next method. Note that when a single
+ * row is larger than this limit the row is still returned completely. The default value is 2MB.
*/
public static final long DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE = 2 * 1024 * 1024;
/**
- * Maximum number of bytes returned when calling a scanner's next method.
- * Note that when a single row is larger than this limit the row is still
- * returned completely.
- * Safety setting to protect the region server.
- *
- * The default value is 100MB. (a client would rarely request larger chunks on purpose)
+ * Maximum number of bytes returned when calling a scanner's next method. Note that when a single
+ * row is larger than this limit the row is still returned completely. Safety setting to protect
+ * the region server. The default value is 100MB. (a client would rarely request larger chunks on
+ * purpose)
*/
public static final long DEFAULT_HBASE_SERVER_SCANNER_MAX_RESULT_SIZE = 100 * 1024 * 1024;
/**
- * Parameter name for client pause value, used mostly as value to wait
- * before running a retry of a failed get, region lookup, etc.
+ * Parameter name for client pause value, used mostly as value to wait before running a retry of a
+ * failed get, region lookup, etc.
*/
public static final String HBASE_CLIENT_PAUSE = "hbase.client.pause";
@@ -693,8 +648,7 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS = 100;
/**
- * The maximum number of concurrent connections the client will maintain to a single
- * RegionServer.
+ * The maximum number of concurrent connections the client will maintain to a single RegionServer.
*/
public static final String HBASE_CLIENT_MAX_PERSERVER_TASKS = "hbase.client.max.perserver.tasks";
@@ -704,8 +658,7 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS = 2;
/**
- * The maximum number of concurrent connections the client will maintain to a single
- * Region.
+ * The maximum number of concurrent connections the client will maintain to a single Region.
*/
public static final String HBASE_CLIENT_MAX_PERREGION_TASKS = "hbase.client.max.perregion.tasks";
@@ -715,8 +668,8 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS = 1;
/**
- * Parameter name for server pause value, used mostly as value to wait before
- * running a retry of a failed operation.
+ * Parameter name for server pause value, used mostly as value to wait before running a retry of a
+ * failed operation.
*/
public static final String HBASE_SERVER_PAUSE = "hbase.server.pause";
@@ -726,9 +679,9 @@ public final class HConstants {
public static final int DEFAULT_HBASE_SERVER_PAUSE = 1000;
/**
- * Parameter name for maximum retries, used as maximum for all retryable
- * operations such as fetching of the root region from root region server,
- * getting a cell's value, starting a row update, etc.
+ * Parameter name for maximum retries, used as maximum for all retryable operations such as
+ * fetching of the root region from root region server, getting a cell's value, starting a row
+ * update, etc.
*/
public static final String HBASE_CLIENT_RETRIES_NUMBER = "hbase.client.retries.number";
@@ -748,10 +701,9 @@ public final class HConstants {
public static final int DEFAULT_HBASE_CLIENT_SCANNER_CACHING = Integer.MAX_VALUE;
/**
- * Parameter name for number of rows that will be fetched when calling next on
- * a scanner if it is not served from memory. Higher caching values will
- * enable faster scanners but will eat up more memory and some calls of next
- * may take longer and longer times when the cache is empty.
+ * Parameter name for number of rows that will be fetched when calling next on a scanner if it is
+ * not served from memory. Higher caching values will enable faster scanners but will eat up more
+ * memory and some calls of next may take longer and longer times when the cache is empty.
*/
public static final String HBASE_META_SCANNER_CACHING = "hbase.meta.scanner.caching";
@@ -918,18 +870,13 @@ public final class HConstants {
public static final String LOCALHOST = "localhost";
/**
- * If this parameter is set to true, then hbase will read
- * data and then verify checksums. Checksum verification
- * inside hdfs will be switched off. However, if the hbase-checksum
- * verification fails, then it will switch back to using
- * hdfs checksums for verifiying data that is being read from storage.
- *
- * If this parameter is set to false, then hbase will not
- * verify any checksums, instead it will depend on checksum verification
- * being done in the hdfs client.
+ * If this parameter is set to true, then hbase will read data and then verify checksums. Checksum
+ * verification inside hdfs will be switched off. However, if the hbase-checksum verification
+ * fails, then it will switch back to using hdfs checksums for verifiying data that is being read
+ * from storage. If this parameter is set to false, then hbase will not verify any checksums,
+ * instead it will depend on checksum verification being done in the hdfs client.
*/
- public static final String HBASE_CHECKSUM_VERIFICATION =
- "hbase.regionserver.checksum.verify";
+ public static final String HBASE_CHECKSUM_VERIFICATION = "hbase.regionserver.checksum.verify";
public static final String LOCALHOST_IP = "127.0.0.1";
@@ -944,17 +891,15 @@ public final class HConstants {
public static final int DEFAULT_REGION_SERVER_HANDLER_COUNT = 30;
/*
- * REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT:
- * -1 => Disable aborting
- * 0 => Abort if even a single handler has died
- * 0.x => Abort only when this percent of handlers have died
- * 1 => Abort only all of the handers have died
+ * REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT: -1 => Disable aborting 0 => Abort if even a
+ * single handler has died 0.x => Abort only when this percent of handlers have died 1 => Abort
+ * only all of the handers have died
*/
public static final String REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT =
"hbase.regionserver.handler.abort.on.error.percent";
public static final double DEFAULT_REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT = 0.5;
- //High priority handlers to deal with admin requests and system table operation requests
+ // High priority handlers to deal with admin requests and system table operation requests
public static final String REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT =
"hbase.regionserver.metahandler.count";
public static final int DEFAULT_REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT = 20;
@@ -977,19 +922,16 @@ public final class HConstants {
public static final int DEFAULT_META_REPLICA_NUM = 1;
/**
- * The name of the configuration parameter that specifies
- * the number of bytes in a newly created checksum chunk.
+ * The name of the configuration parameter that specifies the number of bytes in a newly created
+ * checksum chunk.
*/
- public static final String BYTES_PER_CHECKSUM =
- "hbase.hstore.bytes.per.checksum";
+ public static final String BYTES_PER_CHECKSUM = "hbase.hstore.bytes.per.checksum";
/**
- * The name of the configuration parameter that specifies
- * the name of an algorithm that is used to compute checksums
- * for newly created blocks.
+ * The name of the configuration parameter that specifies the name of an algorithm that is used to
+ * compute checksums for newly created blocks.
*/
- public static final String CHECKSUM_TYPE_NAME =
- "hbase.hstore.checksum.algorithm";
+ public static final String CHECKSUM_TYPE_NAME = "hbase.hstore.checksum.algorithm";
/** Enable file permission modification from standard hbase */
public static final String ENABLE_DATA_FILE_UMASK = "hbase.data.umask.enable";
@@ -997,16 +939,14 @@ public final class HConstants {
public static final String DATA_FILE_UMASK_KEY = "hbase.data.umask";
/** Configuration name of WAL Compression */
- public static final String ENABLE_WAL_COMPRESSION =
- "hbase.regionserver.wal.enablecompression";
-
- /** Configuration name of WAL storage policy
- * Valid values are:
- * NONE: no preference in destination of block replicas
- * ONE_SSD: place only one block replica in SSD and the remaining in default storage
- * and ALL_SSD: place all block replicas on SSD
- *
- * See http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html*/
+ public static final String ENABLE_WAL_COMPRESSION = "hbase.regionserver.wal.enablecompression";
+
+ /**
+ * Configuration name of WAL storage policy Valid values are: NONE: no preference in destination
+ * of block replicas ONE_SSD: place only one block replica in SSD and the remaining in default
+ * storage and ALL_SSD: place all block replicas on SSD See
+ * http://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-hdfs/ArchivalStorage.html
+ */
public static final String WAL_STORAGE_POLICY = "hbase.wal.storage.policy";
public static final String DEFAULT_WAL_STORAGE_POLICY = "NONE";
@@ -1017,17 +957,16 @@ public final class HConstants {
public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
/**
- * The byte array represents for NO_NEXT_INDEXED_KEY;
- * The actual value is irrelevant because this is always compared by reference.
+ * The byte array represents for NO_NEXT_INDEXED_KEY; The actual value is irrelevant because this
+ * is always compared by reference.
*/
public static final Cell NO_NEXT_INDEXED_KEY = new KeyValue();
/** delimiter used between portions of a region name */
public static final int DELIMITER = ',';
/**
- * QOS attributes: these attributes are used to demarcate RPC call processing
- * by different set of handlers. For example, HIGH_QOS tagged methods are
- * handled by high priority handlers.
+ * QOS attributes: these attributes are used to demarcate RPC call processing by different set of
+ * handlers. For example, HIGH_QOS tagged methods are handled by high priority handlers.
*/
// normal_QOS < QOS_threshold < replication_QOS < replay_QOS < admin_QOS < high_QOS
public static final int NORMAL_QOS = 0;
@@ -1042,8 +981,8 @@ public final class HConstants {
public static final String HFILE_ARCHIVE_DIRECTORY = "archive";
/**
- * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for
- * remaining snapshot constants; this is here to keep HConstants dependencies at a minimum and
+ * Name of the directory to store all snapshots. See SnapshotDescriptionUtils for remaining
+ * snapshot constants; this is here to keep HConstants dependencies at a minimum and
* uni-directional.
*/
public static final String SNAPSHOT_DIR_NAME = ".hbase-snapshot";
@@ -1059,49 +998,42 @@ public final class HConstants {
public static final String REGIONSERVER_METRICS_PERIOD = "hbase.regionserver.metrics.period";
public static final long DEFAULT_REGIONSERVER_METRICS_PERIOD = 5000;
/** Directories that are not HBase table directories */
- public static final List<String> HBASE_NON_TABLE_DIRS =
- Collections.unmodifiableList(Arrays.asList(new String[] {
- HBCK_SIDELINEDIR_NAME, HBASE_TEMP_DIRECTORY, MIGRATION_NAME
- }));
+ public static final List<String> HBASE_NON_TABLE_DIRS = Collections.unmodifiableList(Arrays
+ .asList(new String[] { HBCK_SIDELINEDIR_NAME, HBASE_TEMP_DIRECTORY, MIGRATION_NAME }));
/** Directories that are not HBase user table directories */
- public static final List<String> HBASE_NON_USER_TABLE_DIRS =
- Collections.unmodifiableList(Arrays.asList((String[])ArrayUtils.addAll(
- new String[] { TableName.META_TABLE_NAME.getNameAsString() },
- HBASE_NON_TABLE_DIRS.toArray())));
+ public static final List<String> HBASE_NON_USER_TABLE_DIRS = Collections.unmodifiableList(Arrays
+ .asList((String[]) ArrayUtils.addAll(
+ new String[] { TableName.META_TABLE_NAME.getNameAsString() },
+ HBASE_NON_TABLE_DIRS.toArray())));
/** Health script related settings. */
public static final String HEALTH_SCRIPT_LOC = "hbase.node.health.script.location";
public static final String HEALTH_SCRIPT_TIMEOUT = "hbase.node.health.script.timeout";
- public static final String HEALTH_CHORE_WAKE_FREQ =
- "hbase.node.health.script.frequency";
+ public static final String HEALTH_CHORE_WAKE_FREQ = "hbase.node.health.script.frequency";
public static final long DEFAULT_HEALTH_SCRIPT_TIMEOUT = 60000;
/**
* The maximum number of health check failures a server can encounter consecutively.
*/
- public static final String HEALTH_FAILURE_THRESHOLD =
- "hbase.node.health.failure.threshold";
+ public static final String HEALTH_FAILURE_THRESHOLD = "hbase.node.health.failure.threshold";
public static final int DEFAULT_HEALTH_FAILURE_THRESHOLD = 3;
-
/**
- * Setting to activate, or not, the publication of the status by the master. Default
- * notification is by a multicast message.
+ * Setting to activate, or not, the publication of the status by the master. Default notification
+ * is by a multicast message.
*/
public static final String STATUS_PUBLISHED = "hbase.status.published";
public static final boolean STATUS_PUBLISHED_DEFAULT = false;
/**
- * IP to use for the multicast status messages between the master and the clients.
- * The default address is chosen as one among others within the ones suitable for multicast
- * messages.
+ * IP to use for the multicast status messages between the master and the clients. The default
+ * address is chosen as one among others within the ones suitable for multicast messages.
*/
public static final String STATUS_MULTICAST_ADDRESS = "hbase.status.multicast.address.ip";
public static final String DEFAULT_STATUS_MULTICAST_ADDRESS = "226.1.1.3";
/**
- * The address to use for binding the local socket for receiving multicast. Defaults to
- * 0.0.0.0.
+ * The address to use for binding the local socket for receiving multicast. Defaults to 0.0.0.0.
* @see <a href="https://issues.apache.org/jira/browse/HBASE-9961">HBASE-9961</a>
*/
public static final String STATUS_MULTICAST_BIND_ADDRESS =
@@ -1134,7 +1066,7 @@ public final class HConstants {
/** Configuration key for the name of the alternate master key for the cluster, a string */
public static final String CRYPTO_MASTERKEY_ALTERNATE_NAME_CONF_KEY =
- "hbase.crypto.master.alternate.key.name";
+ "hbase.crypto.master.alternate.key.name";
/** Configuration key for the algorithm to use when encrypting the WAL, a string */
public static final String CRYPTO_WAL_ALGORITHM_CONF_KEY = "hbase.crypto.wal.algorithm";
@@ -1166,7 +1098,7 @@ public final class HConstants {
/** Config for pluggable consensus provider */
public static final String HBASE_COORDINATED_STATE_MANAGER_CLASS =
- "hbase.coordinated.state.manager.class";
+ "hbase.coordinated.state.manager.class";
/** Configuration key for SplitLog manager timeout */
public static final String HBASE_SPLITLOG_MANAGER_TIMEOUT = "hbase.splitlog.manager.timeout";
@@ -1180,17 +1112,18 @@ public final class HConstants {
// hbase-common?
/**
- * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path
- * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for
- * list of supported ioengine options.
- * <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
+ * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path to the
+ * file that will host the file-based cache. See BucketCache#getIOEngineFromName() for list of
+ * supported ioengine options.
+ * <p>
+ * Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
*/
public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
/**
* When using bucket cache, this is a float that EITHER represents a percentage of total heap
- * memory size to give to the cache (if < 1.0) OR, it is the capacity in
- * megabytes of the cache.
+ * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the
+ * cache.
*/
public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
@@ -1203,26 +1136,25 @@ public final class HConstants {
public static final String HBASE_CLIENT_FAST_FAIL_MODE_ENABLED =
"hbase.client.fast.fail.mode.enabled";
- public static final boolean HBASE_CLIENT_ENABLE_FAST_FAIL_MODE_DEFAULT =
- false;
+ public static final boolean HBASE_CLIENT_ENABLE_FAST_FAIL_MODE_DEFAULT = false;
public static final String HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS =
"hbase.client.fastfail.threshold";
- public static final long HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT =
- 60000;
+ public static final long HBASE_CLIENT_FAST_FAIL_THREASHOLD_MS_DEFAULT = 60000;
public static final String HBASE_CLIENT_FAST_FAIL_CLEANUP_MS_DURATION_MS =
"hbase.client.fast.fail.cleanup.duration";
- public static final long HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT =
- 600000;
+ public static final long HBASE_CLIENT_FAST_FAIL_CLEANUP_DURATION_MS_DEFAULT = 600000;
public static final String HBASE_CLIENT_FAST_FAIL_INTERCEPTOR_IMPL =
"hbase.client.fast.fail.interceptor.impl";
- /** Config key for if the server should send backpressure and if the client should listen to
- * that backpressure from the server */
+ /**
+ * Config key for if the server should send backpressure and if the client should listen to that
+ * backpressure from the server
+ */
public static final String ENABLE_CLIENT_BACKPRESSURE = "hbase.client.backpressure.enabled";
public static final boolean DEFAULT_ENABLE_CLIENT_BACKPRESSURE = false;
@@ -1234,11 +1166,11 @@ public final class HConstants {
public static final float DEFAULT_HEAP_OCCUPANCY_HIGH_WATERMARK = 0.98f;
/**
- * The max number of threads used for splitting storefiles in parallel during
- * the region split process.
+ * The max number of threads used for splitting storefiles in parallel during the region split
+ * process.
*/
public static final String REGION_SPLIT_THREADS_MAX =
- "hbase.regionserver.region.split.threads.max";
+ "hbase.regionserver.region.split.threads.max";
/** Canary config keys */
public static final String HBASE_CANARY_WRITE_DATA_TTL_KEY = "hbase.canary.write.data.ttl";
@@ -1263,6 +1195,15 @@ public final class HConstants {
public static final String ZK_SERVER_KEYTAB_FILE = "hbase.zookeeper.server.keytab.file";
public static final String ZK_SERVER_KERBEROS_PRINCIPAL =
"hbase.zookeeper.server.kerberos.principal";
+ /**
+ * Backup/Restore constants
+ */
+
+ public final static String BACKUP_ENABLE_KEY = "hbase.backup.enable";
+ public final static boolean BACKUP_ENABLE_DEFAULT = true;
+ public final static String BACKUP_SYSTEM_TTL_KEY = "hbase.backup.system.ttl";
+ // Default TTL = 1 year
+ public final static int BACKUP_SYSTEM_TTL_DEFAULT = 365 * 24 * 3600;
private HConstants() {
// Can't be instantiated with this ctor.
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 26aad71..807b021 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -394,6 +394,11 @@
<version>${project.version}</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop-two.version}</version>
+ </dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
@@ -407,6 +412,11 @@
<artifactId>commons-collections</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop-two.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java
new file mode 100644
index 0000000..28cf627
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupClient.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_COMMAND;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Backup HBase tables locally or on a remote cluster Serve as client entry point for the following
+ * features: - Full Backup provide local and remote back/restore for a list of tables - Incremental
+ * backup to build on top of full backup as daily/weekly backup - Convert incremental backup WAL
+ * files into hfiles - Merge several backup images into one(like merge weekly into monthly) - Add
+ * and remove table to and from Backup image - Cancel a backup process - Full backup based on
+ * existing snapshot - Describe information of a backup image
+ */
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class BackupClient {
+ private static final Log LOG = LogFactory.getLog(BackupClient.class);
+ private static Options opt;
+ private static Configuration conf = null;
+
+ private BackupClient() {
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ protected static void init() throws IOException {
+ // define supported options
+ opt = new Options();
+
+ opt.addOption("all", false, "All tables");
+ opt.addOption("debug", false, "Enable debug loggings");
+ opt.addOption("t", true, "Table name");
+
+ // create configuration instance
+ conf = getConf();
+
+ // disable irrelevant loggers to avoid it mess up command output
+ disableUselessLoggers();
+
+ }
+
+
+ public static void main(String[] args) throws IOException {
+ init();
+ parseAndRun(args);
+ System.exit(0);
+ }
+
+ /**
+ * Set the configuration from a given one.
+ * @param newConf A new given configuration
+ */
+ public synchronized static void setConf(Configuration newConf) {
+ conf = newConf;
+ BackupUtil.setConf(newConf);
+ }
+
+ public static Configuration getConf() {
+ if (conf == null) {
+ conf = BackupUtil.getConf();
+ }
+ return conf;
+ }
+
+ private static void disableUselessLoggers() {
+ // disable zookeeper log to avoid it mess up command output
+ Logger zkLogger = Logger.getLogger("org.apache.zookeeper");
+ LOG.debug("Zookeeper log level before set: " + zkLogger.getLevel());
+ zkLogger.setLevel(Level.OFF);
+ LOG.debug("Zookeeper log level after set: " + zkLogger.getLevel());
+
+ // disable hbase zookeeper tool log to avoid it mess up command output
+ Logger hbaseZkLogger = Logger.getLogger("org.apache.hadoop.hbase.zookeeper");
+ LOG.debug("HBase zookeeper log level before set: " + hbaseZkLogger.getLevel());
+ hbaseZkLogger.setLevel(Level.OFF);
+ LOG.debug("HBase Zookeeper log level after set: " + hbaseZkLogger.getLevel());
+
+ // disable hbase client log to avoid it mess up command output
+ Logger hbaseClientLogger = Logger.getLogger("org.apache.hadoop.hbase.client");
+ LOG.debug("HBase client log level before set: " + hbaseClientLogger.getLevel());
+ hbaseClientLogger.setLevel(Level.OFF);
+ LOG.debug("HBase client log level after set: " + hbaseClientLogger.getLevel());
+ }
+
+ private static void parseAndRun(String[] args) throws IOException {
+
+ String cmd = null;
+ String[] remainArgs = null;
+ if (args == null || args.length == 0) {
+ BackupCommands.createCommand(BackupRestoreConstants.BACKUP_COMMAND.HELP, null).execute();
+ } else {
+ cmd = args[0];
+ remainArgs = new String[args.length - 1];
+ if (args.length > 1) {
+ System.arraycopy(args, 1, remainArgs, 0, args.length - 1);
+ }
+ }
+ CommandLine cmdline = null;
+ try {
+ cmdline = new PosixParser().parse(opt, remainArgs);
+ } catch (ParseException e) {
+ LOG.error("Could not parse command", e);
+ System.exit(-1);
+ }
+
+ BACKUP_COMMAND type = BACKUP_COMMAND.HELP;
+ if (BACKUP_COMMAND.CREATE.name().equalsIgnoreCase(cmd)) {
+ type = BACKUP_COMMAND.CREATE;
+ } else if (BACKUP_COMMAND.HELP.name().equalsIgnoreCase(cmd)) {
+ type = BACKUP_COMMAND.HELP;
+ } else {
+ System.out.println("Unsupported command for backup: " + cmd);
+ }
+
+ // enable debug logging
+ Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup");
+ if (cmdline.hasOption("debug")) {
+ backupClientLogger.setLevel(Level.DEBUG);
+ } else {
+ backupClientLogger.setLevel(Level.INFO);
+ }
+
+ BackupCommands.createCommand(type, cmdline).execute();
+ }
+
+ /**
+ * Send backup request to server, and monitor the progress if necessary
+ * @param backupType : full or incremental
+ * @param backupRootPath : the rooPath specified by user
+ * @param tableListStr : the table list specified by user
+ * @param snapshot : using existing snapshot if specified by user (in future jira)
+ * @return backupId backup id
+ * @throws IOException exception
+ * @throws KeeperException excpetion
+ */
+ public static String create(String backupType, String backupRootPath, String tableListStr,
+ String snapshot) throws IOException {
+
+ String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime();
+
+ // check target path first, confirm it doesn't exist before backup
+ boolean isTargetExist = false;
+ try {
+ isTargetExist = HBackupFileSystem.checkPathExist(backupRootPath, conf);
+ } catch (IOException e) {
+ String expMsg = e.getMessage();
+ String newMsg = null;
+ if (expMsg.contains("No FileSystem for scheme")) {
+ newMsg =
+ "Unsupported filesystem scheme found in the backup target url. Error Message: "
+ + newMsg;
+ LOG.error(newMsg);
+ throw new IOException(newMsg);
+ } else {
+ throw e;
+ }
+ } catch (RuntimeException e) {
+ LOG.error(e.getMessage());
+ throw e;
+ }
+
+ if (isTargetExist) {
+ LOG.info("Using existing backup root dir: " + backupRootPath);
+ } else {
+ LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");
+ }
+
+ // table list specified for backup, trigger backup on specified tables
+ String tableList = tableListStr;
+ // (tableListStr == null) ? null : tableListStr.replaceAll(
+ // BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND,
+ // BackupRestoreConstants.TABLENAME_DELIMITER_IN_ZNODE);
+ try {
+ requestBackup(backupId, backupType, tableList, backupRootPath, snapshot);
+ } catch (RuntimeException e) {
+ String errMsg = e.getMessage();
+ if (errMsg != null
+ && (errMsg.startsWith("Non-existing tables found") || errMsg
+ .startsWith("Snapshot is not found"))) {
+ LOG.error(errMsg + ", please check your command");
+ throw e;
+ } else {
+ throw e;
+ }
+ }
+ return backupId;
+ }
+
+ /**
+ * Prepare and submit Backup request
+ * @param backupId : backup_timestame (something like backup_1398729212626)
+ * @param backupType : full or incremental
+ * @param tableList : tables to be backuped
+ * @param targetRootDir : specified by user
+ * @param snapshot : use existing snapshot if specified by user (for future jira)
+ * @throws IOException exception
+ */
+ protected static void requestBackup(String backupId, String backupType, String tableList,
+ String targetRootDir, String snapshot) throws IOException {
+
+ Configuration conf = getConf();
+ BackupManager backupManager = null;
+ BackupContext backupContext = null;
+ if (snapshot != null) {
+ LOG.warn("Snapshot option specified, backup type and table option will be ignored,\n"
+ + "full backup will be taken based on the given snapshot.");
+ throw new IOException("backup using existing Snapshot will be implemented in future jira");
+ }
+
+ HBaseAdmin hbadmin = null;
+ Connection conn = null;
+ try {
+
+ backupManager = new BackupManager(conf);
+ String tables = tableList;
+ if (backupType.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ Set<String> incrTableSet = backupManager.getIncrementalBackupTableSet();
+ if (incrTableSet.isEmpty()) {
+ LOG.warn("Incremental backup table set contains no table.\n"
+ + "Use 'backup create full' or 'backup stop' to \n "
+ + "change the tables covered by incremental backup.");
+ throw new RuntimeException("No table covered by incremental backup.");
+ }
+ StringBuilder sb = new StringBuilder();
+ for (String tableName : incrTableSet) {
+ sb.append(tableName + " ");
+ }
+ LOG.info("Incremental backup for the following table set: " + sb.toString());
+ tables =
+ sb.toString().trim()
+ .replaceAll(" ", BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+ }
+
+ // check whether table exists first before starting real request
+ if (tables != null) {
+ String[] tableNames = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+ ArrayList<String> noneExistingTableList = null;
+ conn = ConnectionFactory.createConnection(conf);
+ hbadmin = (HBaseAdmin) conn.getAdmin();
+ for (String tableName : tableNames) {
+ if (!hbadmin.tableExists(TableName.valueOf(tableName))) {
+ if (noneExistingTableList == null) {
+ noneExistingTableList = new ArrayList<String>();
+ }
+ noneExistingTableList.add(tableName);
+ }
+ }
+ if (noneExistingTableList != null) {
+ if (backupType.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ LOG.warn("Incremental backup table set contains no-exising table: "
+ + noneExistingTableList);
+ } else {
+ // Throw exception only in full mode - we try to backup non-existing table
+ throw new RuntimeException("Non-existing tables found in the table list: "
+ + noneExistingTableList);
+ }
+ }
+ }
+
+ // if any target table backup dir already exist, then no backup action taken
+ String[] tableNames = null;
+ if (tables != null && !tables.equals("")) {
+ tableNames = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+ }
+ if (tableNames != null && tableNames.length > 0) {
+ for (String table : tableNames) {
+ String targetTableBackupDir =
+ HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
+ Path targetTableBackupDirPath = new Path(targetTableBackupDir);
+ FileSystem outputFs = FileSystem.get(targetTableBackupDirPath.toUri(), conf);
+ if (outputFs.exists(targetTableBackupDirPath)) {
+ throw new IOException("Target backup directory " + targetTableBackupDir
+ + " exists already.");
+ }
+ }
+ }
+ backupContext =
+ backupManager.createBackupContext(backupId, backupType, tables, targetRootDir, snapshot);
+ backupManager.initialize();
+ backupManager.dispatchRequest(backupContext);
+ } catch (BackupException e) {
+ // suppress the backup exception wrapped within #initialize or #dispatchRequest, backup
+ // exception has already been handled normally
+ StackTraceElement[] stes = e.getStackTrace();
+ for (StackTraceElement ste : stes) {
+ LOG.info(ste);
+ }
+ LOG.error("Backup Exception " + e.getMessage());
+ } finally {
+ if (hbadmin != null) {
+ hbadmin.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java
new file mode 100644
index 0000000..c7aff15
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCommands.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_COMMAND;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * General backup commands, options and usage messages
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+final class BackupCommands {
+
+ private static final String USAGE = "Usage: hbase backup COMMAND\n"
+ + "where COMMAND is one of:\n" + " create create a new backup image\n"
+ + "Enter \'help COMMAND\' to see help message for each command\n";
+
+ private static final String CREATE_CMD_USAGE =
+ "Usage: hbase backup create <type> <backup_root_path> [tables] [-s name] [-convert] "
+ + "[-silent]\n" + " type \"full\" to create a full backup image;\n"
+ + " \"incremental\" to create an incremental backup image\n"
+ + " backup_root_path The full root path to store the backup image,\n"
+ + " the prefix can be gpfs, hdfs or webhdfs\n" + " Options:\n"
+ + " tables If no tables (\"\") are specified, all tables are backed up. "
+ + "Otherwise it is a\n" + " comma separated list of tables.\n"
+ + " -s name Use the specified snapshot for full backup\n"
+ + " -convert For an incremental backup, convert WAL files to HFiles\n";
+
+ interface Command {
+ void execute() throws IOException;
+ }
+
+ private BackupCommands() {
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ static Command createCommand(BACKUP_COMMAND type, CommandLine cmdline) {
+ Command cmd = null;
+ switch (type) {
+ case CREATE:
+ cmd = new CreateCommand(cmdline);
+ break;
+ case HELP:
+ default:
+ cmd = new HelpCommand(cmdline);
+ break;
+ }
+ return cmd;
+ }
+
+ private static class CreateCommand implements Command {
+ CommandLine cmdline;
+
+ CreateCommand(CommandLine cmdline) {
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ if (cmdline == null || cmdline.getArgs() == null) {
+ System.out.println("ERROR: missing arguments");
+ System.out.println(CREATE_CMD_USAGE);
+ System.exit(-1);
+ }
+ String[] args = cmdline.getArgs();
+ if (args.length < 2 || args.length > 3) {
+ System.out.println("ERROR: wrong number of arguments");
+ System.out.println(CREATE_CMD_USAGE);
+ System.exit(-1);
+ }
+
+ if (!BackupRestoreConstants.BACKUP_TYPE_FULL.equalsIgnoreCase(args[0])
+ && !BackupRestoreConstants.BACKUP_TYPE_INCR.equalsIgnoreCase(args[0])) {
+ System.out.println("ERROR: invalid backup type");
+ System.out.println(CREATE_CMD_USAGE);
+ System.exit(-1);
+ }
+
+ String snapshot = cmdline.hasOption('s') ? cmdline.getOptionValue('s') : null;
+ String tables = (args.length == 3) ? args[2] : null;
+
+ try {
+ BackupClient.create(args[0], args[1], tables, snapshot);
+ } catch (RuntimeException e) {
+ System.out.println("ERROR: " + e.getMessage());
+ System.exit(-1);
+ }
+ }
+ }
+
+ private static class HelpCommand implements Command {
+ CommandLine cmdline;
+
+ HelpCommand(CommandLine cmdline) {
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ if (cmdline == null) {
+ System.out.println(USAGE);
+ System.exit(0);
+ }
+
+ String[] args = cmdline.getArgs();
+ if (args == null || args.length == 0) {
+ System.out.println(USAGE);
+ System.exit(0);
+ }
+
+ if (args.length != 1) {
+ System.out.println("Only support check help message of a single command type");
+ System.out.println(USAGE);
+ System.exit(0);
+ }
+
+ String type = args[0];
+
+ if (BACKUP_COMMAND.CREATE.name().equalsIgnoreCase(type)) {
+ System.out.println(CREATE_CMD_USAGE);
+ } // other commands will be supported in future jira
+ System.exit(0);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java
new file mode 100644
index 0000000..e8d6444
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupContext.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * An object to encapsulate the information for each backup request
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupContext implements Serializable {
+
+ private static final long serialVersionUID = 2401435114454300992L;
+
+ // backup id: a timestamp when we request the backup
+ private String backupId;
+
+ // backup type, full or incremental
+ private String type;
+
+ // target root directory for storing the backup files
+ private String targetRootDir;
+
+ // overall backup status
+ private BackupHandler.BACKUPSTATUS flag;
+
+ // overall backup phase
+ private BackupHandler.BACKUPPHASE phase;
+
+ // overall backup failure message
+ private String failedMsg;
+
+ // backup status map for all tables
+ private Map<String, BackupStatus> backupStatusMap;
+
+ // actual start timestamp of the backup process
+ private long startTs;
+
+ // actual end timestamp of the backup process, could be fail or complete
+ private long endTs;
+
+ // the total bytes of incremental logs copied
+ private long totalBytesCopied;
+
+ // for incremental backup, the location of the backed-up hlogs
+ private String hlogTargetDir = null;
+
+ // incremental backup file list
+ transient private List<String> incrBackupFileList;
+
+ // new region server log timestamps for table set after distributed log roll
+ // key - table name, value - map of RegionServer hostname -> last log rolled timestamp
+ transient private HashMap<String, HashMap<String, String>> tableSetTimestampMap;
+
+ // cancel flag
+ private boolean cancelled = false;
+ // backup progress string
+
+ private String progress;
+
+ public BackupContext() {
+ }
+
+ public BackupContext(String backupId, String type, String[] tables, String targetRootDir,
+ String snapshot) {
+ super();
+
+ if (backupStatusMap == null) {
+ backupStatusMap = new HashMap<String, BackupStatus>();
+ }
+
+ this.backupId = backupId;
+ this.type = type;
+ this.targetRootDir = targetRootDir;
+
+ this.addTables(tables);
+
+ if (type.equals(BackupRestoreConstants.BACKUP_TYPE_INCR)) {
+ setHlogTargetDir(HBackupFileSystem.getLogBackupDir(targetRootDir, backupId));
+ }
+
+ this.startTs = 0;
+ this.endTs = 0;
+
+ }
+
+ /**
+ * Set progress string
+ * @param msg progress message
+ */
+
+ public void setProgress(String msg) {
+ this.progress = msg;
+ }
+
+ /**
+ * Get current progress msg
+ */
+ public String getProgress() {
+ return progress;
+ }
+
+ /**
+ * Mark cancel flag.
+ */
+ public void markCancel() {
+ this.cancelled = true;
+ }
+
+ /**
+ * Has been marked as cancelled or not.
+ * @return True if marked as cancelled
+ */
+ public boolean isCancelled() {
+ return this.cancelled;
+ }
+
+ public String getBackupId() {
+ return backupId;
+ }
+
+ public void setBackupId(String backupId) {
+ this.backupId = backupId;
+ }
+
+ public BackupStatus getBackupStatus(String table) {
+ return this.backupStatusMap.get(table);
+ }
+
+ public String getFailedMsg() {
+ return failedMsg;
+ }
+
+ public void setFailedMsg(String failedMsg) {
+ this.failedMsg = failedMsg;
+ }
+
+ public long getStartTs() {
+ return startTs;
+ }
+
+ public void setStartTs(long startTs) {
+ this.startTs = startTs;
+ }
+
+ public long getEndTs() {
+ return endTs;
+ }
+
+ public void setEndTs(long endTs) {
+ this.endTs = endTs;
+ }
+
+ public long getTotalBytesCopied() {
+ return totalBytesCopied;
+ }
+
+ public BackupHandler.BACKUPSTATUS getFlag() {
+ return flag;
+ }
+
+ public void setFlag(BackupHandler.BACKUPSTATUS flag) {
+ this.flag = flag;
+ }
+
+ public BackupHandler.BACKUPPHASE getPhase() {
+ return phase;
+ }
+
+ public void setPhase(BackupHandler.BACKUPPHASE phase) {
+ this.phase = phase;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setSnapshotName(String table, String snapshotName) {
+ this.backupStatusMap.get(table).setSnapshotName(snapshotName);
+ }
+
+ public String getSnapshotName(String table) {
+ return this.backupStatusMap.get(table).getSnapshotName();
+ }
+
+ public List<String> getSnapshotNames() {
+ List<String> snapshotNames = new ArrayList<String>();
+ for (BackupStatus backupStatus : this.backupStatusMap.values()) {
+ snapshotNames.add(backupStatus.getSnapshotName());
+ }
+ return snapshotNames;
+ }
+
+ public Set<String> getTables() {
+ return this.backupStatusMap.keySet();
+ }
+
+ public String getTableListAsString() {
+ return BackupUtil.concat(backupStatusMap.keySet(), ";");
+ }
+
+ public void addTables(String[] tables) {
+ for (String table : tables) {
+ BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId);
+ this.backupStatusMap.put(table, backupStatus);
+ }
+ }
+
+ public String getTargetRootDir() {
+ return targetRootDir;
+ }
+
+ public void setHlogTargetDir(String hlogTagetDir) {
+ this.hlogTargetDir = hlogTagetDir;
+ }
+
+ public String getHLogTargetDir() {
+ return hlogTargetDir;
+ }
+
+ public List<String> getIncrBackupFileList() {
+ return incrBackupFileList;
+ }
+
+ public List<String> setIncrBackupFileList(List<String> incrBackupFileList) {
+ this.incrBackupFileList = incrBackupFileList;
+ return this.incrBackupFileList;
+ }
+
+ /**
+ * Set the new region server log timestamps after distributed log roll
+ * @param newTableSetTimestampMap table timestamp map
+ */
+ public void setIncrTimestampMap(HashMap<String,
+ HashMap<String, String>> newTableSetTimestampMap) {
+ this.tableSetTimestampMap = newTableSetTimestampMap;
+ }
+
+ /**
+ * Get new region server log timestamps after distributed log roll
+ * @return new region server log timestamps
+ */
+ public HashMap<String, HashMap<String, String>> getIncrTimestampMap() {
+ return this.tableSetTimestampMap;
+ }
+
+ /**
+ * Get existing snapshot if backing up from existing snapshot.
+ * @return The existing snapshot, null if not backing up from existing snapshot
+ */
+ public String getExistingSnapshot() {
+ // this feature will be supported in another Jira
+ return null;
+ }
+
+ /**
+ * Check whether this backup context are for backing up from existing snapshot or not.
+ * @return true if it is for backing up from existing snapshot, otherwise false
+ */
+ public boolean fromExistingSnapshot() {
+ // this feature will be supported in later jiras
+ return false;
+ }
+
+ public String getTableBySnapshot(String snapshotName) {
+ for (Entry<String, BackupStatus> entry : this.backupStatusMap.entrySet()) {
+ if (snapshotName.equals(entry.getValue().getSnapshotName())) {
+ return entry.getKey();
+ }
+ }
+ return null;
+ }
+
+ public byte[] toByteArray() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+ oos.writeObject(this);
+ return baos.toByteArray();
+ }
+
+ public static BackupContext fromByteArray(byte[] data)
+ throws IOException, ClassNotFoundException {
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ ObjectInputStream ois = new ObjectInputStream(bais);
+ return (BackupContext) ois.readObject();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java
new file mode 100644
index 0000000..223746d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupCopyService.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface BackupCopyService extends Configurable {
+ static enum Type {
+ FULL, INCREMENTAL
+ }
+
+ public int copy(BackupHandler backupHandler, Configuration conf, BackupCopyService.Type copyType,
+ String[] options) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/de69f0df/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java
new file mode 100644
index 0000000..fe0d0e2
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupException.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Backup exception
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupException extends HBaseIOException {
+ private BackupContext description;
+
+ /**
+ * Some exception happened for a backup and don't even know the backup that it was about
+ * @param msg Full description of the failure
+ */
+ public BackupException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * Some exception happened for a backup with a cause
+ * @param cause the cause
+ */
+ public BackupException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Exception for the given backup that has no previous root cause
+ * @param msg reason why the backup failed
+ * @param desc description of the backup that is being failed
+ */
+ public BackupException(String msg, BackupContext desc) {
+ super(msg);
+ this.description = desc;
+ }
+
+ /**
+ * Exception for the given backup due to another exception
+ * @param msg reason why the backup failed
+ * @param cause root cause of the failure
+ * @param desc description of the backup that is being failed
+ */
+ public BackupException(String msg, Throwable cause, BackupContext desc) {
+ super(msg, cause);
+ this.description = desc;
+ }
+
+ /**
+ * Exception when the description of the backup cannot be determined, due to some other root
+ * cause
+ * @param message description of what caused the failure
+ * @param e root cause
+ */
+ public BackupException(String message, Exception e) {
+ super(message, e);
+ }
+
+ public BackupContext getBackupContext() {
+ return this.description;
+ }
+
+}
[17/29] hbase git commit: HBASE-14867 SimpleRegionNormalizer needs to
have better heuristics to trigger merge operation
Posted by sy...@apache.org.
HBASE-14867 SimpleRegionNormalizer needs to have better heuristics to trigger merge operation
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e4992c6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e4992c6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e4992c6
Branch: refs/heads/hbase-12439
Commit: 1e4992c6eccb81166cdda842a68644fa962a3fdc
Parents: 6868c63
Author: tedyu <yu...@gmail.com>
Authored: Tue Dec 29 06:21:57 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Tue Dec 29 06:21:57 2015 -0800
----------------------------------------------------------------------
.../normalizer/SimpleRegionNormalizer.java | 106 +++++++++----------
.../normalizer/TestSimpleRegionNormalizer.java | 43 +++++++-
2 files changed, 90 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e4992c6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 659b3dc..fe6034b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -27,8 +27,11 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Triple;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
/**
@@ -66,6 +69,22 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
this.masterServices = masterServices;
}
+ /*
+ * This comparator compares the region size.
+ * The second element in the triple is region size while the 3rd element
+ * is the index of the region in the underlying List
+ */
+ private Comparator<Triple<HRegionInfo, Long, Integer>> regionSizeComparator =
+ new Comparator<Triple<HRegionInfo, Long, Integer>>() {
+ @Override
+ public int compare(Triple<HRegionInfo, Long, Integer> pair,
+ Triple<HRegionInfo, Long, Integer> pair2) {
+ long sz = pair.getSecond();
+ long sz2 = pair2.getSecond();
+ return (sz < sz2) ? -1 : ((sz == sz2) ? 0 : 1);
+ }
+ };
+
/**
* Computes next most "urgent" normalization action on the table.
* Action may be either a split, or a merge, or no action.
@@ -76,7 +95,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
@Override
public NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException {
if (table == null || table.isSystemTable()) {
- LOG.debug("Normalization of table " + table + " isn't allowed");
+ LOG.debug("Normalization of system table " + table + " isn't allowed");
return EmptyNormalizationPlan.getInstance();
}
@@ -95,57 +114,18 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
", number of regions: " + tableRegions.size());
long totalSizeMb = 0;
- Pair<HRegionInfo, Long> largestRegion = new Pair<>();
-
- // A is a smallest region, B is it's smallest neighbor
- Pair<HRegionInfo, Long> smallestRegion = new Pair<>();
- int smallestRegionIndex = 0;
+ ArrayList<Triple<HRegionInfo, Long, Integer>> regionsWithSize =
+ new ArrayList<Triple<HRegionInfo, Long, Integer>>(tableRegions.size());
for (int i = 0; i < tableRegions.size(); i++) {
HRegionInfo hri = tableRegions.get(i);
long regionSize = getRegionSize(hri);
+ regionsWithSize.add(new Triple<HRegionInfo, Long, Integer>(hri, regionSize, i));
totalSizeMb += regionSize;
-
- if (largestRegion.getFirst() == null || regionSize > largestRegion.getSecond()) {
- largestRegion.setFirst(hri);
- largestRegion.setSecond(regionSize);
- }
-
- if (smallestRegion.getFirst() == null || regionSize < smallestRegion.getSecond()) {
- smallestRegion.setFirst(hri);
- smallestRegion.setSecond(regionSize);
- smallestRegionIndex = i;
- }
}
+ Collections.sort(regionsWithSize, regionSizeComparator);
- // now get smallest neighbor of smallest region
- long leftNeighborSize = -1;
-
- if (smallestRegionIndex > 0) {
- leftNeighborSize = getRegionSize(tableRegions.get(smallestRegionIndex - 1));
- }
-
- long rightNeighborSize = -1;
- if (smallestRegionIndex < tableRegions.size() - 1) {
- rightNeighborSize = getRegionSize(tableRegions.get(smallestRegionIndex + 1));
- }
-
- Pair<HRegionInfo, Long> smallestNeighborOfSmallestRegion;
- if (leftNeighborSize == -1) {
- smallestNeighborOfSmallestRegion =
- new Pair<>(tableRegions.get(smallestRegionIndex + 1), rightNeighborSize);
- } else if (rightNeighborSize == -1) {
- smallestNeighborOfSmallestRegion =
- new Pair<>(tableRegions.get(smallestRegionIndex - 1), leftNeighborSize);
- } else {
- if (leftNeighborSize < rightNeighborSize) {
- smallestNeighborOfSmallestRegion =
- new Pair<>(tableRegions.get(smallestRegionIndex - 1), leftNeighborSize);
- } else {
- smallestNeighborOfSmallestRegion =
- new Pair<>(tableRegions.get(smallestRegionIndex + 1), rightNeighborSize);
- }
- }
+ Triple<HRegionInfo, Long, Integer> largestRegion = regionsWithSize.get(tableRegions.size()-1);
double avgRegionSize = totalSizeMb / (double) tableRegions.size();
@@ -159,19 +139,31 @@ public class SimpleRegionNormalizer implements RegionNormalizer {
+ largestRegion.getFirst().getRegionNameAsString() + " has size "
+ largestRegion.getSecond() + ", more than 2 times than avg size, splitting");
return new SplitNormalizationPlan(largestRegion.getFirst(), null);
- } else {
- if (smallestRegion.getSecond() + smallestNeighborOfSmallestRegion.getSecond()
- < avgRegionSize) {
- LOG.debug("Table " + table + ", smallest region size: " + smallestRegion.getSecond()
- + " and its smallest neighbor size: " + smallestNeighborOfSmallestRegion.getSecond()
- + ", less than the avg size, merging them");
- return new MergeNormalizationPlan(smallestRegion.getFirst(),
- smallestNeighborOfSmallestRegion.getFirst());
- } else {
- LOG.debug("No normalization needed, regions look good for table: " + table);
- return EmptyNormalizationPlan.getInstance();
+ }
+ int candidateIdx = 0;
+ // look for two successive entries whose indices are adjacent
+ while (candidateIdx < tableRegions.size()-1) {
+ if (Math.abs(regionsWithSize.get(candidateIdx).getThird() -
+ regionsWithSize.get(candidateIdx + 1).getThird()) == 1) {
+ break;
}
+ candidateIdx++;
+ }
+ if (candidateIdx == tableRegions.size()-1) {
+ LOG.debug("No neighboring regions found for table: " + table);
+ return EmptyNormalizationPlan.getInstance();
+ }
+ Triple<HRegionInfo, Long, Integer> candidateRegion = regionsWithSize.get(candidateIdx);
+ Triple<HRegionInfo, Long, Integer> candidateRegion2 = regionsWithSize.get(candidateIdx+1);
+ if (candidateRegion.getSecond() + candidateRegion2.getSecond() < avgRegionSize) {
+ LOG.debug("Table " + table + ", smallest region size: " + candidateRegion.getSecond()
+ + " and its smallest neighbor size: " + candidateRegion2.getSecond()
+ + ", less than the avg size, merging them");
+ return new MergeNormalizationPlan(candidateRegion.getFirst(),
+ candidateRegion2.getFirst());
}
+ LOG.debug("No normalization needed, regions look good for table: " + table);
+ return EmptyNormalizationPlan.getInstance();
}
private long getRegionSize(HRegionInfo hri) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e4992c6/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
index 211911e..970af43 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java
@@ -142,8 +142,8 @@ public class TestSimpleRegionNormalizer {
hris.add(hri4);
regionSizes.put(hri4.getRegionName(), 15);
- HRegionInfo hri5 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
- hris.add(hri4);
+ HRegionInfo hri5 = new HRegionInfo(testTable, Bytes.toBytes("eee"), Bytes.toBytes("fff"));
+ hris.add(hri5);
regionSizes.put(hri5.getRegionName(), 16);
setupMocksForNormalizer(regionSizes, hris);
@@ -154,6 +154,45 @@ public class TestSimpleRegionNormalizer {
assertEquals(hri3, ((MergeNormalizationPlan) plan).getSecondRegion());
}
+ // Test for situation illustrated in HBASE-14867
+ @Test
+ public void testMergeOfSecondSmallestRegions() throws HBaseIOException {
+ TableName testTable = TableName.valueOf("testMergeOfSmallRegions");
+ List<HRegionInfo> hris = new ArrayList<>();
+ Map<byte[], Integer> regionSizes = new HashMap<>();
+
+ HRegionInfo hri1 = new HRegionInfo(testTable, Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
+ hris.add(hri1);
+ regionSizes.put(hri1.getRegionName(), 1);
+
+ HRegionInfo hri2 = new HRegionInfo(testTable, Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
+ hris.add(hri2);
+ regionSizes.put(hri2.getRegionName(), 10000);
+
+ HRegionInfo hri3 = new HRegionInfo(testTable, Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
+ hris.add(hri3);
+ regionSizes.put(hri3.getRegionName(), 10000);
+
+ HRegionInfo hri4 = new HRegionInfo(testTable, Bytes.toBytes("ddd"), Bytes.toBytes("eee"));
+ hris.add(hri4);
+ regionSizes.put(hri4.getRegionName(), 10000);
+
+ HRegionInfo hri5 = new HRegionInfo(testTable, Bytes.toBytes("eee"), Bytes.toBytes("fff"));
+ hris.add(hri5);
+ regionSizes.put(hri5.getRegionName(), 2700);
+
+ HRegionInfo hri6 = new HRegionInfo(testTable, Bytes.toBytes("fff"), Bytes.toBytes("ggg"));
+ hris.add(hri6);
+ regionSizes.put(hri6.getRegionName(), 2700);
+
+ setupMocksForNormalizer(regionSizes, hris);
+ NormalizationPlan plan = normalizer.computePlanForTable(testTable);
+
+ assertTrue(plan instanceof MergeNormalizationPlan);
+ assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion());
+ assertEquals(hri6, ((MergeNormalizationPlan) plan).getSecondRegion());
+ }
+
@Test
public void testMergeOfSmallNonAdjacentRegions() throws HBaseIOException {
TableName testTable = TableName.valueOf("testMergeOfSmallRegions");
[24/29] hbase git commit: HBASE-15060 Cull TestHFileV2 and
HFileWriterFactory
Posted by sy...@apache.org.
HBASE-15060 Cull TestHFileV2 and HFileWriterFactory
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/92abf8ac
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/92abf8ac
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/92abf8ac
Branch: refs/heads/hbase-12439
Commit: 92abf8ac5743849d4c32d6abf8fd33f161bab249
Parents: c1b6d47
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Tue Dec 29 17:41:04 2015 -0800
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Wed Dec 30 13:21:54 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 10 +-
.../hbase/io/hfile/HFileWriterFactory.java | 40 ---
.../hbase/io/hfile/RandomKeyValueUtil.java | 104 ++++++
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 6 +-
.../hbase/io/hfile/TestHFileBlockIndex.java | 10 +-
.../TestHFileInlineToRootChunkConversion.java | 2 +-
.../hbase/io/hfile/TestHFileWriterV2.java | 342 -------------------
.../hbase/io/hfile/TestHFileWriterV3.java | 9 +-
.../hfile/TestLazyDataBlockDecompression.java | 6 +-
.../hadoop/hbase/io/hfile/TestPrefetch.java | 4 +-
.../hfile/TestSeekBeforeWithInlineBlocks.java | 6 +-
.../regionserver/TestCacheOnWriteInSchema.java | 6 +-
.../regionserver/TestCompoundBloomFilter.java | 12 +-
13 files changed, 137 insertions(+), 420 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 35b232a..669ad92 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -241,7 +241,7 @@ public class HFile {
* This variety of ways to construct writers is used throughout the code, and
* we want to be able to swap writer implementations.
*/
- public static abstract class WriterFactory {
+ public static class WriterFactory {
protected final Configuration conf;
protected final CacheConfig cacheConf;
protected FileSystem fs;
@@ -309,12 +309,8 @@ public class HFile {
else if (LOG.isDebugEnabled()) LOG.debug("Unable to set drop behind on " + path);
}
}
- return createWriter(fs, path, ostream,
- comparator, fileContext);
+ return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, fileContext);
}
-
- protected abstract Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
- CellComparator comparator, HFileContext fileContext) throws IOException;
}
/** The configuration key for HFile version to use for new files */
@@ -351,7 +347,7 @@ public class HFile {
" hfiles only (but it can read v2 files without having to update hfile.format.version " +
"in hbase-site.xml)");
case 3:
- return new HFileWriterFactory(conf, cacheConf);
+ return new HFile.WriterFactory(conf, cacheConf);
default:
throw new IllegalArgumentException("Cannot create writer for HFile " +
"format version " + version);
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java
deleted file mode 100644
index 92eb760..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitationsME
- * under the License.
- */
-package org.apache.hadoop.hbase.io.hfile;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.CellComparator;
-
-public class HFileWriterFactory extends HFile.WriterFactory {
- HFileWriterFactory(Configuration conf, CacheConfig cacheConf) {
- super(conf, cacheConf);
- }
-
- @Override
- public HFile.Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream,
- CellComparator comparator, HFileContext context)
- throws IOException {
- return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, context);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java
new file mode 100644
index 0000000..9507345
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomKeyValueUtil.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+import java.util.Random;
+
+/**
+ * These helper methods generate random byte[]'s data for KeyValues
+ */
+public class RandomKeyValueUtil {
+ public static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_";
+ private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64;
+ private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128;
+
+ public static final char randomReadableChar(Random rand) {
+ int i = rand.nextInt(26 * 2 + 10 + 1);
+ if (i < 26)
+ return (char) ('A' + i);
+ i -= 26;
+
+ if (i < 26)
+ return (char) ('a' + i);
+ i -= 26;
+
+ if (i < 10)
+ return (char) ('0' + i);
+ i -= 10;
+
+ assert i == 0;
+ return '_';
+ }
+
+ public static KeyValue randomKeyValue(Random rand) {
+ return new KeyValue(randomRowOrQualifier(rand),
+ COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand),
+ randomValue(rand));
+ }
+
+ public static byte[] randomRowOrQualifier(Random rand) {
+ StringBuilder field = new StringBuilder();
+ int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
+ + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
+ - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
+ for (int i = 0; i < fieldLen; ++i)
+ field.append(randomReadableChar(rand));
+ return field.toString().getBytes();
+ }
+
+ public static byte[] randomValue(Random rand) {
+ StringBuilder v = new StringBuilder();
+ for (int j = 0; j < 1 + rand.nextInt(2000); ++j) {
+ v.append((char) (32 + rand.nextInt(95)));
+ }
+
+ byte[] valueBytes = v.toString().getBytes();
+ return valueBytes;
+ }
+
+ /**
+ * Generates a random key that is guaranteed to increase as the given index i
+ * increases. The result consists of a prefix, which is a deterministic
+ * increasing function of i, and a random suffix.
+ *
+ * @param rand
+ * random number generator to use
+ * @param i
+ * @return
+ */
+ public static byte[] randomOrderedKey(Random rand, int i) {
+ StringBuilder k = new StringBuilder();
+
+ // The fixed-length lexicographically increasing part of the key.
+ for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
+ if ((i & (1 << bitIndex)) == 0)
+ k.append("a");
+ else
+ k.append("b");
+ }
+
+ // A random-length random suffix of the key.
+ for (int j = 0; j < rand.nextInt(50); ++j)
+ k.append(randomReadableChar(rand));
+
+ byte[] keyBytes = k.toString().getBytes();
+ return keyBytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index 2c957ef..ccf59a4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -380,9 +380,9 @@ public class TestCacheOnWrite {
.withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build();
byte[] cf = Bytes.toBytes("fam");
for (int i = 0; i < NUM_KV; ++i) {
- byte[] row = TestHFileWriterV2.randomOrderedKey(rand, i);
- byte[] qualifier = TestHFileWriterV2.randomRowOrQualifier(rand);
- byte[] value = TestHFileWriterV2.randomValue(rand);
+ byte[] row = RandomKeyValueUtil.randomOrderedKey(rand, i);
+ byte[] qualifier = RandomKeyValueUtil.randomRowOrQualifier(rand);
+ byte[] value = RandomKeyValueUtil.randomValue(rand);
KeyValue kv;
if(useTags) {
Tag t = new Tag((byte) 1, "visibility");
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index 8a843a3..687d3cd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -280,7 +280,7 @@ public class TestHFileBlockIndex {
byte[] qualifier = Bytes.toBytes("q");
for (int j = 0; j < 16; ++j) {
byte[] k =
- new KeyValue(TestHFileWriterV2.randomOrderedKey(rand, i * 16 + j), family, qualifier,
+ new KeyValue(RandomKeyValueUtil.randomOrderedKey(rand, i * 16 + j), family, qualifier,
EnvironmentEdgeManager.currentTime(), KeyValue.Type.Put).getKey();
keys.add(k);
if (j == 8) {
@@ -348,7 +348,7 @@ public class TestHFileBlockIndex {
int secondaryIndexEntries[] = new int[numTotalKeys];
for (int i = 0; i < numTotalKeys; ++i) {
- byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i * 2);
+ byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i * 2);
KeyValue cell = new KeyValue(k, Bytes.toBytes("f"), Bytes.toBytes("q"),
Bytes.toBytes("val"));
//KeyValue cell = new KeyValue.KeyOnlyKeyValue(k, 0, k.length);
@@ -473,7 +473,7 @@ public class TestHFileBlockIndex {
c.writeRoot(dos);
assertEquals(c.getRootSize(), dos.size());
- byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i);
+ byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i);
numSubEntries += rand.nextInt(5) + 1;
keys.add(k);
c.add(k, getDummyFileOffset(i), getDummyOnDiskSize(i), numSubEntries);
@@ -556,12 +556,12 @@ public class TestHFileBlockIndex {
byte[] family = Bytes.toBytes("f");
byte[] qualifier = Bytes.toBytes("q");
for (int i = 0; i < NUM_KV; ++i) {
- byte[] row = TestHFileWriterV2.randomOrderedKey(rand, i);
+ byte[] row = RandomKeyValueUtil.randomOrderedKey(rand, i);
// Key will be interpreted by KeyValue.KEY_COMPARATOR
KeyValue kv =
new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(),
- TestHFileWriterV2.randomValue(rand));
+ RandomKeyValueUtil.randomValue(rand));
byte[] k = kv.getKey();
writer.append(kv);
keys[i] = k;
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
index ab811f1..af4f2b8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java
@@ -55,7 +55,7 @@ public class TestHFileInlineToRootChunkConversion {
CacheConfig cacheConf = new CacheConfig(conf);
conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize);
HFileContext context = new HFileContextBuilder().withBlockSize(16).build();
- HFile.Writer hfw = new HFileWriterFactory(conf, cacheConf)
+ HFile.Writer hfw = new HFile.WriterFactory(conf, cacheConf)
.withFileContext(context)
.withPath(fs, hfPath).create();
List<byte[]> keys = new ArrayList<byte[]>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
deleted file mode 100644
index 975073a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.io.hfile;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
-import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
-import org.apache.hadoop.hbase.nio.ByteBuff;
-import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableUtils;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Testing writing a version 2 {@link HFile}. This is a low-level test written
- * during the development of {@link HFileWriterImpl}.
- */
-@Category({IOTests.class, SmallTests.class})
-public class TestHFileWriterV2 {
-
- private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class);
-
- private static final HBaseTestingUtility TEST_UTIL =
- new HBaseTestingUtility();
-
- private Configuration conf;
- private FileSystem fs;
-
- @Before
- public void setUp() throws IOException {
- conf = TEST_UTIL.getConfiguration();
- fs = FileSystem.get(conf);
- }
-
- @Test
- public void testHFileFormatV2() throws IOException {
- Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV2");
- final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ;
- final int entryCount = 10000;
- writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false);
- }
-
- @Test
- public void testMidKeyInHFile() throws IOException{
- Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
- "testMidKeyInHFile");
- Compression.Algorithm compressAlgo = Compression.Algorithm.NONE;
- int entryCount = 50000;
- writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true);
- }
-
- private void writeDataAndReadFromHFile(Path hfilePath,
- Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException {
-
- HFileContext context = new HFileContextBuilder()
- .withBlockSize(4096)
- .withCompression(compressAlgo)
- .build();
- HFile.Writer writer = new HFileWriterFactory(conf, new CacheConfig(conf))
- .withPath(fs, hfilePath)
- .withFileContext(context)
- .create();
-
- Random rand = new Random(9713312); // Just a fixed seed.
- List<KeyValue> keyValues = new ArrayList<KeyValue>(entryCount);
-
- for (int i = 0; i < entryCount; ++i) {
- byte[] keyBytes = randomOrderedKey(rand, i);
-
- // A random-length random value.
- byte[] valueBytes = randomValue(rand);
- KeyValue keyValue = new KeyValue(keyBytes, null, null, valueBytes);
- writer.append(keyValue);
- keyValues.add(keyValue);
- }
-
- // Add in an arbitrary order. They will be sorted lexicographically by
- // the key.
- writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C."));
- writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow"));
- writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris"));
-
- writer.close();
-
-
- FSDataInputStream fsdis = fs.open(hfilePath);
-
- // A "manual" version of a new-format HFile reader. This unit test was
- // written before the V2 reader was fully implemented.
-
- long fileSize = fs.getFileStatus(hfilePath).getLen();
- FixedFileTrailer trailer =
- FixedFileTrailer.readFromStream(fsdis, fileSize);
-
- assertEquals(entryCount, trailer.getEntryCount());
-
- HFileContext meta = new HFileContextBuilder()
- .withHBaseCheckSum(true)
- .withIncludesMvcc(false)
- .withIncludesTags(false)
- .withCompression(compressAlgo)
- .build();
-
- HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(fsdis, fileSize, meta);
- // Comparator class name is stored in the trailer in version 2.
- CellComparator comparator = trailer.createComparator();
- HFileBlockIndex.BlockIndexReader dataBlockIndexReader =
- new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator,
- trailer.getNumDataIndexLevels());
- HFileBlockIndex.BlockIndexReader metaBlockIndexReader =
- new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
-
- HFileBlock.BlockIterator blockIter = blockReader.blockRange(
- trailer.getLoadOnOpenDataOffset(),
- fileSize - trailer.getTrailerSize());
- // Data index. We also read statistics about the block index written after
- // the root level.
- dataBlockIndexReader.readMultiLevelIndexRoot(
- blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
- trailer.getDataIndexCount());
-
- if (findMidKey) {
- Cell midkey = dataBlockIndexReader.midkey();
- assertNotNull("Midkey should not be null", midkey);
- }
-
- // Meta index.
- metaBlockIndexReader.readRootIndex(
- blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX)
- .getByteStream(), trailer.getMetaIndexCount());
- // File info
- FileInfo fileInfo = new FileInfo();
- fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
- byte [] keyValueFormatVersion = fileInfo.get(HFileWriterImpl.KEY_VALUE_VERSION);
- boolean includeMemstoreTS = keyValueFormatVersion != null &&
- Bytes.toInt(keyValueFormatVersion) > 0;
-
- // Counters for the number of key/value pairs and the number of blocks
- int entriesRead = 0;
- int blocksRead = 0;
- long memstoreTS = 0;
-
- // Scan blocks the way the reader would scan them
- fsdis.seek(0);
- long curBlockPos = 0;
- while (curBlockPos <= trailer.getLastDataBlockOffset()) {
- HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false);
- assertEquals(BlockType.DATA, block.getBlockType());
- if (meta.isCompressedOrEncrypted()) {
- assertFalse(block.isUnpacked());
- block = block.unpack(meta, blockReader);
- }
- ByteBuff buf = block.getBufferWithoutHeader();
- while (buf.hasRemaining()) {
- int keyLen = buf.getInt();
- int valueLen = buf.getInt();
-
- byte[] key = new byte[keyLen];
- buf.get(key);
-
- byte[] value = new byte[valueLen];
- buf.get(value);
-
- if (includeMemstoreTS) {
- ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(),
- buf.arrayOffset() + buf.position(), buf.remaining());
- DataInputStream data_input = new DataInputStream(byte_input);
-
- memstoreTS = WritableUtils.readVLong(data_input);
- buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS));
- }
-
- // A brute-force check to see that all keys and values are correct.
- KeyValue kv = keyValues.get(entriesRead);
- assertTrue(Bytes.compareTo(key, kv.getKey()) == 0);
- assertTrue(Bytes.compareTo(value, 0, value.length, kv.getValueArray(), kv.getValueOffset(),
- kv.getValueLength()) == 0);
-
- ++entriesRead;
- }
- ++blocksRead;
- curBlockPos += block.getOnDiskSizeWithHeader();
- }
- LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead="
- + blocksRead);
- assertEquals(entryCount, entriesRead);
-
- // Meta blocks. We can scan until the load-on-open data offset (which is
- // the root block index offset in version 2) because we are not testing
- // intermediate-level index blocks here.
-
- int metaCounter = 0;
- while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) {
- LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " +
- trailer.getLoadOnOpenDataOffset());
- HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false)
- .unpack(meta, blockReader);
- assertEquals(BlockType.META, block.getBlockType());
- Text t = new Text();
- ByteBuff buf = block.getBufferWithoutHeader();
- if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) {
- throw new IOException("Failed to deserialize block " + this + " into a " + t.getClass().getSimpleName());
- }
- Text expectedText =
- (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text(
- "Moscow") : new Text("Washington, D.C."));
- assertEquals(expectedText, t);
- LOG.info("Read meta block data: " + t);
- ++metaCounter;
- curBlockPos += block.getOnDiskSizeWithHeader();
- }
-
- fsdis.close();
- }
-
-
- // Static stuff used by various HFile v2 unit tests
-
- public static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_";
- private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64;
- private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128;
-
- /**
- * Generates a random key that is guaranteed to increase as the given index i
- * increases. The result consists of a prefix, which is a deterministic
- * increasing function of i, and a random suffix.
- *
- * @param rand
- * random number generator to use
- * @param i
- * @return
- */
- public static byte[] randomOrderedKey(Random rand, int i) {
- StringBuilder k = new StringBuilder();
-
- // The fixed-length lexicographically increasing part of the key.
- for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
- if ((i & (1 << bitIndex)) == 0)
- k.append("a");
- else
- k.append("b");
- }
-
- // A random-length random suffix of the key.
- for (int j = 0; j < rand.nextInt(50); ++j)
- k.append(randomReadableChar(rand));
-
- byte[] keyBytes = k.toString().getBytes();
- return keyBytes;
- }
-
- public static byte[] randomValue(Random rand) {
- StringBuilder v = new StringBuilder();
- for (int j = 0; j < 1 + rand.nextInt(2000); ++j) {
- v.append((char) (32 + rand.nextInt(95)));
- }
-
- byte[] valueBytes = v.toString().getBytes();
- return valueBytes;
- }
-
- public static final char randomReadableChar(Random rand) {
- int i = rand.nextInt(26 * 2 + 10 + 1);
- if (i < 26)
- return (char) ('A' + i);
- i -= 26;
-
- if (i < 26)
- return (char) ('a' + i);
- i -= 26;
-
- if (i < 10)
- return (char) ('0' + i);
- i -= 10;
-
- assert i == 0;
- return '_';
- }
-
- public static byte[] randomRowOrQualifier(Random rand) {
- StringBuilder field = new StringBuilder();
- int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
- + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
- - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
- for (int i = 0; i < fieldLen; ++i)
- field.append(randomReadableChar(rand));
- return field.toString().getBytes();
- }
-
- public static KeyValue randomKeyValue(Random rand) {
- return new KeyValue(randomRowOrQualifier(rand),
- COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand),
- randomValue(rand));
- }
-
-
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
index 979c9f6..15aa912 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
@@ -120,7 +120,7 @@ public class TestHFileWriterV3 {
.withBlockSize(4096)
.withIncludesTags(useTags)
.withCompression(compressAlgo).build();
- HFile.Writer writer = new HFileWriterFactory(conf, new CacheConfig(conf))
+ HFile.Writer writer = new HFile.WriterFactory(conf, new CacheConfig(conf))
.withPath(fs, hfilePath)
.withFileContext(context)
.withComparator(CellComparator.COMPARATOR)
@@ -130,10 +130,10 @@ public class TestHFileWriterV3 {
List<KeyValue> keyValues = new ArrayList<KeyValue>(entryCount);
for (int i = 0; i < entryCount; ++i) {
- byte[] keyBytes = TestHFileWriterV2.randomOrderedKey(rand, i);
+ byte[] keyBytes = RandomKeyValueUtil.randomOrderedKey(rand, i);
// A random-length random value.
- byte[] valueBytes = TestHFileWriterV2.randomValue(rand);
+ byte[] valueBytes = RandomKeyValueUtil.randomValue(rand);
KeyValue keyValue = null;
if (useTags) {
ArrayList<Tag> tags = new ArrayList<Tag>();
@@ -176,7 +176,7 @@ public class TestHFileWriterV3 {
.withHBaseCheckSum(true).build();
HFileBlock.FSReader blockReader =
new HFileBlock.FSReaderImpl(fsdis, fileSize, meta);
- // Comparator class name is stored in the trailer in version 2.
+ // Comparator class name is stored in the trailer in version 3.
CellComparator comparator = trailer.createComparator();
HFileBlockIndex.BlockIndexReader dataBlockIndexReader =
new HFileBlockIndex.CellBasedKeyBlockIndexReader(comparator,
@@ -297,6 +297,5 @@ public class TestHFileWriterV3 {
fsdis.close();
}
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java
index 0067417..5f73500 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLazyDataBlockDecompression.java
@@ -89,7 +89,7 @@ public class TestLazyDataBlockDecompression {
*/
private static void writeHFile(Configuration conf, CacheConfig cc, FileSystem fs, Path path,
HFileContext cxt, int entryCount) throws IOException {
- HFile.Writer writer = new HFileWriterFactory(conf, cc)
+ HFile.Writer writer = new HFile.WriterFactory(conf, cc)
.withPath(fs, path)
.withFileContext(cxt)
.create();
@@ -100,8 +100,8 @@ public class TestLazyDataBlockDecompression {
final byte[] qualifier = Bytes.toBytes("q");
for (int i = 0; i < entryCount; i++) {
- byte[] keyBytes = TestHFileWriterV2.randomOrderedKey(rand, i);
- byte[] valueBytes = TestHFileWriterV2.randomValue(rand);
+ byte[] keyBytes = RandomKeyValueUtil.randomOrderedKey(rand, i);
+ byte[] valueBytes = RandomKeyValueUtil.randomValue(rand);
// make a real keyvalue so that hfile tool can examine it
writer.append(new KeyValue(keyBytes, family, qualifier, valueBytes));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index 053a27e..69a77bf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -111,8 +111,8 @@ public class TestPrefetch {
final int rowLen = 32;
for (int i = 0; i < NUM_KV; ++i) {
- byte[] k = TestHFileWriterV2.randomOrderedKey(RNG, i);
- byte[] v = TestHFileWriterV2.randomValue(RNG);
+ byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i);
+ byte[] v = RandomKeyValueUtil.randomValue(RNG);
int cfLen = RNG.nextInt(k.length - rowLen + 1);
KeyValue kv = new KeyValue(
k, 0, rowLen,
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java
index c9af3d3..0dbb4e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekBeforeWithInlineBlocks.java
@@ -123,9 +123,9 @@ public class TestSeekBeforeWithInlineBlocks {
.build();
for (int i = 0; i < NUM_KV; i++) {
- byte[] row = TestHFileWriterV2.randomOrderedKey(RAND, i);
- byte[] qual = TestHFileWriterV2.randomRowOrQualifier(RAND);
- byte[] value = TestHFileWriterV2.randomValue(RAND);
+ byte[] row = RandomKeyValueUtil.randomOrderedKey(RAND, i);
+ byte[] qual = RandomKeyValueUtil.randomRowOrQualifier(RAND);
+ byte[] value = RandomKeyValueUtil.randomValue(RAND);
KeyValue kv = new KeyValue(row, FAM, qual, value);
storeFileWriter.append(kv);
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
index da3c17d..0916fe6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.RandomKeyValueUtil;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.TableName;
@@ -49,7 +50,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
-import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.util.Bytes;
@@ -276,8 +276,8 @@ public class TestCacheOnWriteInSchema {
private void writeStoreFile(StoreFile.Writer writer) throws IOException {
final int rowLen = 32;
for (int i = 0; i < NUM_KV; ++i) {
- byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i);
- byte[] v = TestHFileWriterV2.randomValue(rand);
+ byte[] k = RandomKeyValueUtil.randomOrderedKey(rand, i);
+ byte[] v = RandomKeyValueUtil.randomValue(rand);
int cfLen = rand.nextInt(k.length - rowLen + 1);
KeyValue kv = new KeyValue(
k, 0, rowLen,
http://git-wip-us.apache.org/repos/asf/hbase/blob/92abf8ac/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
index 8300210..44e249d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.io.hfile.RandomKeyValueUtil;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.client.Scan;
@@ -53,7 +54,6 @@ import org.apache.hadoop.hbase.io.hfile.CompoundBloomFilterWriter;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
-import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.BloomFilterUtil;
@@ -144,7 +144,7 @@ public class TestCompoundBloomFilter {
private List<KeyValue> createSortedKeyValues(Random rand, int n) {
List<KeyValue> kvList = new ArrayList<KeyValue>(n);
for (int i = 0; i < n; ++i)
- kvList.add(TestHFileWriterV2.randomKeyValue(rand));
+ kvList.add(RandomKeyValueUtil.randomKeyValue(rand));
Collections.sort(kvList, CellComparator.COMPARATOR);
return kvList;
}
@@ -231,7 +231,7 @@ public class TestCompoundBloomFilter {
Random rand = new Random(EVALUATION_SEED);
int nTrials = NUM_KV[t] * 10;
for (int i = 0; i < nTrials; ++i) {
- byte[] query = TestHFileWriterV2.randomRowOrQualifier(rand);
+ byte[] query = RandomKeyValueUtil.randomRowOrQualifier(rand);
if (isInBloom(scanner, query, bt, rand)) {
numFalsePos += 1;
}
@@ -280,16 +280,16 @@ public class TestCompoundBloomFilter {
private boolean isInBloom(StoreFileScanner scanner, byte[] row, BloomType bt,
Random rand) {
- return isInBloom(scanner, row, TestHFileWriterV2.randomRowOrQualifier(rand));
+ return isInBloom(scanner, row, RandomKeyValueUtil.randomRowOrQualifier(rand));
}
private boolean isInBloom(StoreFileScanner scanner, byte[] row,
byte[] qualifier) {
Scan scan = new Scan(row, row);
- scan.addColumn(Bytes.toBytes(TestHFileWriterV2.COLUMN_FAMILY_NAME), qualifier);
+ scan.addColumn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME), qualifier);
Store store = mock(Store.class);
HColumnDescriptor hcd = mock(HColumnDescriptor.class);
- when(hcd.getName()).thenReturn(Bytes.toBytes(TestHFileWriterV2.COLUMN_FAMILY_NAME));
+ when(hcd.getName()).thenReturn(Bytes.toBytes(RandomKeyValueUtil.COLUMN_FAMILY_NAME));
when(store.getFamily()).thenReturn(hcd);
return scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
}
[18/29] hbase git commit: HBASE-15043 region_status.rb broken with
TypeError: no public constructors for
Java::OrgApacheHadoopHbaseClient::HBaseAdmin (Samir Ahmic)
Posted by sy...@apache.org.
HBASE-15043 region_status.rb broken with TypeError: no public constructors for Java::OrgApacheHadoopHbaseClient::HBaseAdmin (Samir Ahmic)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9b8895ba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9b8895ba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9b8895ba
Branch: refs/heads/hbase-12439
Commit: 9b8895ba295ee66fefda115f30ab727590e487e8
Parents: 1e4992c
Author: tedyu <yu...@gmail.com>
Authored: Tue Dec 29 07:06:43 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Tue Dec 29 07:06:43 2015 -0800
----------------------------------------------------------------------
bin/draining_servers.rb | 5 ++++-
bin/region_status.rb | 7 ++++---
bin/replication/copy_tables_desc.rb | 12 +++++++++---
bin/shutdown_regionserver.rb | 6 +++++-
4 files changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9b8895ba/bin/draining_servers.rb
----------------------------------------------------------------------
diff --git a/bin/draining_servers.rb b/bin/draining_servers.rb
index 45c9694..6849605 100644
--- a/bin/draining_servers.rb
+++ b/bin/draining_servers.rb
@@ -22,6 +22,7 @@ require 'optparse'
include Java
import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.client.ConnectionFactory
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.zookeeper.ZKUtil
import org.apache.commons.logging.Log
@@ -60,6 +61,7 @@ end
def getServerNames(hostOrServers, config)
ret = []
+ connection = ConnectionFactory.createConnection(config)
for hostOrServer in hostOrServers
# check whether it is already serverName. No need to connect to cluster
@@ -67,7 +69,7 @@ def getServerNames(hostOrServers, config)
if parts.size() == 3
ret << hostOrServer
else
- admin = HBaseAdmin.new(config) if not admin
+ admin = connection.getAdmin() if not admin
servers = getServers(admin)
hostOrServer = hostOrServer.gsub(/:/, ",")
@@ -78,6 +80,7 @@ def getServerNames(hostOrServers, config)
end
admin.close() if admin
+ connection.close()
return ret
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/9b8895ba/bin/region_status.rb
----------------------------------------------------------------------
diff --git a/bin/region_status.rb b/bin/region_status.rb
index a0f9d1b..55bc672 100644
--- a/bin/region_status.rb
+++ b/bin/region_status.rb
@@ -67,12 +67,11 @@ org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level)
config = HBaseConfiguration.create
config.set 'fs.defaultFS', config.get(HConstants::HBASE_DIR)
connection = ConnectionFactory.createConnection(config)
-
# wait until the master is running
admin = nil
while true
begin
- admin = HBaseAdmin.new config
+ admin = connection.getAdmin()
break
rescue MasterNotRunningException => e
print 'Waiting for master to start...\n'
@@ -119,7 +118,7 @@ while iter.hasNext
# Gone too far, break
break
end
- region = HRegionInfo.getHRegionInfo result
+ region = MetaTableAccessor::getHRegionInfo(result)
if not region.isOffline
# only include regions that should be online
meta_count += 1
@@ -151,5 +150,7 @@ while true
break
end
end
+admin.close()
+connection.close()
exit server_count == meta_count ? 0 : 1
http://git-wip-us.apache.org/repos/asf/hbase/blob/9b8895ba/bin/replication/copy_tables_desc.rb
----------------------------------------------------------------------
diff --git a/bin/replication/copy_tables_desc.rb b/bin/replication/copy_tables_desc.rb
index bc70031..8a6c670 100644
--- a/bin/replication/copy_tables_desc.rb
+++ b/bin/replication/copy_tables_desc.rb
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.EmptyWatcher
import org.apache.hadoop.hbase.client.HBaseAdmin
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.client.ConnectionFactory
# Name of this script
NAME = "copy_tables_desc"
@@ -56,18 +57,23 @@ c1.set(HConstants::ZOOKEEPER_QUORUM, parts1[0])
c1.set("hbase.zookeeper.property.clientPort", parts1[1])
c1.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts1[2])
-admin1 = HBaseAdmin.new(c1)
+connection1 = ConnectionFactory.createConnection(c1)
+admin1 = connection1.getAdmin()
c2 = HBaseConfiguration.create()
c2.set(HConstants::ZOOKEEPER_QUORUM, parts2[0])
c2.set("hbase.zookeeper.property.clientPort", parts2[1])
c2.set(HConstants::ZOOKEEPER_ZNODE_PARENT, parts2[2])
-admin2 = HBaseAdmin.new(c2)
+connection2 = ConnectionFactory.createConnection(c2)
+admin2 = connection2.getAdmin()
for t in admin1.listTables()
admin2.createTable(t)
end
-
puts "All descriptions were copied"
+admin1.close()
+admin2.close()
+connection1.close()
+connection2.close()
http://git-wip-us.apache.org/repos/asf/hbase/blob/9b8895ba/bin/shutdown_regionserver.rb
----------------------------------------------------------------------
diff --git a/bin/shutdown_regionserver.rb b/bin/shutdown_regionserver.rb
index 9287e5b..608248f 100644
--- a/bin/shutdown_regionserver.rb
+++ b/bin/shutdown_regionserver.rb
@@ -25,6 +25,7 @@
include Java
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.HBaseAdmin
+import org.apache.hadoop.hbase.client.ConnectionFactory
def usage(msg=nil)
$stderr.puts 'Usage: shutdown_regionserver.rb <host:port>..'
@@ -41,8 +42,9 @@ ARGV.each do |x|
end
config = HBaseConfiguration.create()
+connection = ConnectionFactory.createConnection(config)
begin
- admin = HBaseAdmin.new(config)
+ admin = connection.getAdmin()
rescue
abort "Error: Couldn't instantiate HBaseAdmin"
end
@@ -50,3 +52,5 @@ end
ARGV.each do |hostport|
admin.stopRegionServer(hostport)
end
+admin.close()
+connection.close()
[20/29] hbase git commit: HBASE-15007 update docs to list Hadoop
2.6.1+ as cool.
Posted by sy...@apache.org.
HBASE-15007 update docs to list Hadoop 2.6.1+ as cool.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e2c2e17
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e2c2e17
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e2c2e17
Branch: refs/heads/hbase-12439
Commit: 5e2c2e1780d57f2e0086b10cd28d5212d6a3d5d0
Parents: 822fead
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Dec 29 11:46:17 2015 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Dec 29 11:46:17 2015 -0600
----------------------------------------------------------------------
src/main/asciidoc/_chapters/configuration.adoc | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e2c2e17/src/main/asciidoc/_chapters/configuration.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/configuration.adoc b/src/main/asciidoc/_chapters/configuration.adoc
index e7deea3..495232f 100644
--- a/src/main/asciidoc/_chapters/configuration.adoc
+++ b/src/main/asciidoc/_chapters/configuration.adoc
@@ -222,7 +222,8 @@ Use the following legend to interpret this table:
|Hadoop-2.3.x | NT | S | NT | NT | NT
|Hadoop-2.4.x | NT | S | S | S | S
|Hadoop-2.5.x | NT | S | S | S | S
-|Hadoop-2.6.x | X | X | X | X | X
+|Hadoop-2.6.0 | X | X | X | X | X
+|Hadoop-2.6.1+ | NT | NT | NT | NT | S
|Hadoop-2.7.0 | X | X | X | X | X
|Hadoop-2.7.1+ | NT | NT | NT | NT | S
|===
@@ -233,7 +234,7 @@ Use the following legend to interpret this table:
Hadoop distributions based on the 2.6.x line *must* have
link:https://issues.apache.org/jira/browse/HADOOP-11710[HADOOP-11710] applied if you plan to run
HBase on top of an HDFS Encryption Zone. Failure to do so will result in cluster failure and
-data loss.
+data loss. This patch is present in Apache Hadoop releases 2.6.1+.
====
.Hadoop 2.7.x
[29/29] hbase git commit: HBASE-14987 Compaction marker whose region
name doesn't match current region's needs to be handled
Posted by sy...@apache.org.
HBASE-14987 Compaction marker whose region name doesn't match current region's needs to be handled
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/00656688
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/00656688
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/00656688
Branch: refs/heads/hbase-12439
Commit: 00656688f73c85ea9e6f5241ac852f72e774eeea
Parents: 9589a7d
Author: tedyu <yu...@gmail.com>
Authored: Mon Jan 4 07:10:10 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Mon Jan 4 07:10:10 2016 -0800
----------------------------------------------------------------------
.../hadoop/hbase/protobuf/ProtobufUtil.java | 9 +++-
.../hadoop/hbase/regionserver/HRegion.java | 44 ++++++++++++++++----
.../hadoop/hbase/regionserver/TestHRegion.java | 26 ++++++++++--
3 files changed, 65 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/00656688/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index b2d5994..c02309b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -2563,12 +2563,19 @@ public final class ProtobufUtil {
@SuppressWarnings("deprecation")
public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] family,
List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
+ return toCompactionDescriptor(info, null, family, inputPaths, outputPaths, storeDir);
+ }
+
+ @SuppressWarnings("deprecation")
+ public static CompactionDescriptor toCompactionDescriptor(HRegionInfo info, byte[] regionName,
+ byte[] family, List<Path> inputPaths, List<Path> outputPaths, Path storeDir) {
// compaction descriptor contains relative paths.
// input / output paths are relative to the store dir
// store dir is relative to region dir
CompactionDescriptor.Builder builder = CompactionDescriptor.newBuilder()
.setTableName(ByteStringer.wrap(info.getTable().toBytes()))
- .setEncodedRegionName(ByteStringer.wrap(info.getEncodedNameAsBytes()))
+ .setEncodedRegionName(ByteStringer.wrap(
+ regionName == null ? info.getEncodedNameAsBytes() : regionName))
.setFamilyName(ByteStringer.wrap(family))
.setStoreHomeDir(storeDir.getName()); //make relative
for (Path inputPath : inputPaths) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/00656688/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 9549a13..ccf2eb0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4130,11 +4130,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
continue;
}
}
+ boolean checkRowWithinBoundary = false;
// Check this edit is for this region.
if (!Bytes.equals(key.getEncodedRegionName(),
this.getRegionInfo().getEncodedNameAsBytes())) {
- skippedEdits++;
- continue;
+ checkRowWithinBoundary = true;
}
boolean flush = false;
@@ -4142,11 +4142,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Check this edit is for me. Also, guard against writing the special
// METACOLUMN info such as HBASE::CACHEFLUSH entries
if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) {
- //this is a special edit, we should handle it
- CompactionDescriptor compaction = WALEdit.getCompaction(cell);
- if (compaction != null) {
- //replay the compaction
- replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);
+ // if region names don't match, skipp replaying compaction marker
+ if (!checkRowWithinBoundary) {
+ //this is a special edit, we should handle it
+ CompactionDescriptor compaction = WALEdit.getCompaction(cell);
+ if (compaction != null) {
+ //replay the compaction
+ replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);
+ }
}
skippedEdits++;
continue;
@@ -4162,6 +4165,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
skippedEdits++;
continue;
}
+ if (checkRowWithinBoundary && !rowIsInRange(this.getRegionInfo(),
+ cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())) {
+ LOG.warn("Row of " + cell + " is not within region boundary");
+ skippedEdits++;
+ continue;
+ }
// Now, figure if we should skip this edit.
if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getFamily()
.getName())) {
@@ -4232,8 +4241,16 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
void replayWALCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,
boolean removeFiles, long replaySeqId)
throws IOException {
- checkTargetRegion(compaction.getEncodedRegionName().toByteArray(),
- "Compaction marker from WAL ", compaction);
+ try {
+ checkTargetRegion(compaction.getEncodedRegionName().toByteArray(),
+ "Compaction marker from WAL ", compaction);
+ } catch (WrongRegionException wre) {
+ if (RegionReplicaUtil.isDefaultReplica(this.getRegionInfo())) {
+ // skip the compaction marker since it is not for this region
+ return;
+ }
+ throw wre;
+ }
synchronized (writestate) {
if (replaySeqId < lastReplayedOpenRegionSeqId) {
@@ -6590,6 +6607,15 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
(Bytes.compareTo(info.getEndKey(), row) > 0));
}
+ public static boolean rowIsInRange(HRegionInfo info, final byte [] row, final int offset,
+ final short length) {
+ return ((info.getStartKey().length == 0) ||
+ (Bytes.compareTo(info.getStartKey(), 0, info.getStartKey().length,
+ row, offset, length) <= 0)) &&
+ ((info.getEndKey().length == 0) ||
+ (Bytes.compareTo(info.getEndKey(), 0, info.getEndKey().length, row, offset, length) > 0));
+ }
+
/**
* Merge two HRegions. The regions must be adjacent and must not overlap.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/00656688/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 35de488..4582e31 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -839,6 +839,10 @@ public class TestHRegion {
@Test
public void testRecoveredEditsReplayCompaction() throws Exception {
+ testRecoveredEditsReplayCompaction(false);
+ testRecoveredEditsReplayCompaction(true);
+ }
+ public void testRecoveredEditsReplayCompaction(boolean mismatchedRegionName) throws Exception {
String method = name.getMethodName();
TableName tableName = TableName.valueOf(method);
byte[] family = Bytes.toBytes("family");
@@ -884,9 +888,17 @@ public class TestHRegion {
Path newFile = region.getRegionFileSystem().commitStoreFile(Bytes.toString(family),
files[0].getPath());
+ byte[] encodedNameAsBytes = this.region.getRegionInfo().getEncodedNameAsBytes();
+ byte[] fakeEncodedNameAsBytes = new byte [encodedNameAsBytes.length];
+ for (int i=0; i < encodedNameAsBytes.length; i++) {
+ // Mix the byte array to have a new encodedName
+ fakeEncodedNameAsBytes[i] = (byte) (encodedNameAsBytes[i] + 1);
+ }
+
CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(this.region
- .getRegionInfo(), family, storeFiles, Lists.newArrayList(newFile), region
- .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
+ .getRegionInfo(), mismatchedRegionName ? fakeEncodedNameAsBytes : null, family,
+ storeFiles, Lists.newArrayList(newFile),
+ region.getRegionFileSystem().getStoreDir(Bytes.toString(family)));
WALUtil.writeCompactionMarker(region.getWAL(), this.region.getTableDesc(),
this.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
@@ -908,14 +920,20 @@ public class TestHRegion {
region.getTableDesc();
region.getRegionInfo();
region.close();
- region = HRegion.openHRegion(region, null);
+ try {
+ region = HRegion.openHRegion(region, null);
+ } catch (WrongRegionException wre) {
+ fail("Matching encoded region name should not have produced WrongRegionException");
+ }
// now check whether we have only one store file, the compacted one
Collection<StoreFile> sfs = region.getStore(family).getStorefiles();
for (StoreFile sf : sfs) {
LOG.info(sf.getPath());
}
- assertEquals(1, region.getStore(family).getStorefilesCount());
+ if (!mismatchedRegionName) {
+ assertEquals(1, region.getStore(family).getStorefilesCount());
+ }
files = FSUtils.listStatus(fs, tmpDir);
assertTrue("Expected to find 0 files inside " + tmpDir, files == null || files.length == 0);