You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2013/03/04 12:24:53 UTC
svn commit: r1452257 [3/14] - in /hbase/branches/0.94:
security/src/main/java/org/apache/hadoop/hbase/security/access/
security/src/test/java/org/apache/hadoop/hbase/security/access/
src/main/jamon/org/apache/hadoop/hbase/tmpl/master/ src/main/java/org...
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileLinkCleaner.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+
+/**
+ * HFileLink cleaner that determines if a hfile should be deleted.
+ * HFiles can be deleted only if there're no links to them.
+ *
+ * When a HFileLink is created a back reference file is created in:
+ * /hbase/archive/table/region/cf/.links-hfile/ref-region.ref-table
+ * To check if the hfile can be deleted the back references folder must be empty.
+ */
+@InterfaceAudience.Private
+public class HFileLinkCleaner extends BaseHFileCleanerDelegate {
+ private static final Log LOG = LogFactory.getLog(HFileLinkCleaner.class);
+
+ private FileSystem fs = null;
+
+ @Override
+ public synchronized boolean isFileDeletable(Path filePath) {
+ if (this.fs == null) return false;
+
+ // HFile Link is always deletable
+ if (HFileLink.isHFileLink(filePath)) return true;
+
+ // If the file is inside a link references directory, means that is a back ref link.
+ // The back ref can be deleted only if the referenced file doesn't exists.
+ Path parentDir = filePath.getParent();
+ if (HFileLink.isBackReferencesDir(parentDir)) {
+ try {
+ Path hfilePath = HFileLink.getHFileFromBackReference(getConf(), filePath);
+ return !fs.exists(hfilePath);
+ } catch (IOException e) {
+ LOG.error("Couldn't verify if the referenced file still exists, keep it just in case");
+ return false;
+ }
+ }
+
+ // HFile is deletable only if has no links
+ try {
+ Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
+ return FSUtils.listStatus(fs, backRefDir) == null;
+ } catch (IOException e) {
+ LOG.error("Couldn't get the references, not deleting file, just in case");
+ return false;
+ }
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+
+ // setup filesystem
+ try {
+ this.fs = FileSystem.get(this.getConf());
+ } catch (IOException e) {
+ LOG.error("Couldn't instantiate the file system, not deleting file, just in case");
+ }
+ }
+}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java Mon Mar 4 11:24:50 2013
@@ -1,5 +1,4 @@
/**
- * Copyright 2011 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -20,13 +19,25 @@
package org.apache.hadoop.hbase.master.handler;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
@@ -41,21 +52,23 @@ import org.apache.hadoop.hbase.master.As
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.zookeeper.KeeperException;
/**
* Handler to create a table.
*/
+@InterfaceAudience.Private
public class CreateTableHandler extends EventHandler {
private static final Log LOG = LogFactory.getLog(CreateTableHandler.class);
- private MasterFileSystem fileSystemManager;
- private final HTableDescriptor hTableDescriptor;
- private Configuration conf;
- private final AssignmentManager assignmentManager;
- private final CatalogTracker catalogTracker;
- private final ServerManager serverManager;
+ protected MasterFileSystem fileSystemManager;
+ protected final HTableDescriptor hTableDescriptor;
+ protected Configuration conf;
+ protected final AssignmentManager assignmentManager;
+ protected final CatalogTracker catalogTracker;
+ protected final ServerManager serverManager;
private final HRegionInfo [] newRegions;
public CreateTableHandler(Server server, MasterFileSystem fileSystemManager,
@@ -98,8 +111,7 @@ public class CreateTableHandler extends
// table in progress. This will introduce a new zookeeper call. Given
// createTable isn't a frequent operation, that should be ok.
try {
- if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(
- tableName))
+ if (!this.assignmentManager.getZKTable().checkAndSetEnablingTable(tableName))
throw new TableExistsException(tableName);
} catch (KeeperException e) {
throw new IOException("Unable to ensure that the table will be" +
@@ -122,66 +134,91 @@ public class CreateTableHandler extends
public void process() {
String tableName = this.hTableDescriptor.getNameAsString();
try {
- LOG.info("Attemping to create the table " + tableName);
- handleCreateTable();
- } catch (IOException e) {
- LOG.error("Error trying to create the table " + tableName, e);
- } catch (KeeperException e) {
+ LOG.info("Attempting to create the table " + tableName);
+ handleCreateTable(tableName);
+ completed(null);
+ } catch (Throwable e) {
LOG.error("Error trying to create the table " + tableName, e);
+ completed(e);
}
}
- private void handleCreateTable() throws IOException, KeeperException {
-
- // TODO: Currently we make the table descriptor and as side-effect the
- // tableDir is created. Should we change below method to be createTable
- // where we create table in tmp dir with its table descriptor file and then
- // do rename to move it into place?
- FSTableDescriptors.createTableDescriptor(this.hTableDescriptor, this.conf);
-
- List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
- final int batchSize =
- this.conf.getInt("hbase.master.createtable.batchsize", 100);
- for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
- HRegionInfo newRegion = this.newRegions[regionIdx];
- // 1. Create HRegion
- HRegion region = HRegion.createHRegion(newRegion,
- this.fileSystemManager.getRootDir(), this.conf,
- this.hTableDescriptor, null, false, true);
-
- regionInfos.add(region.getRegionInfo());
- if (regionIdx % batchSize == 0) {
- // 2. Insert into META
- MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
- regionInfos.clear();
- }
+ /**
+ * Called after that process() is completed.
+ * @param exception null if process() is successful or not null if something has failed.
+ */
+ protected void completed(final Throwable exception) {
+ }
- // 3. Close the new region to flush to disk. Close log file too.
- region.close();
+ /**
+ * Responsible of table creation (on-disk and META) and assignment.
+ * - Create the table directory and descriptor (temp folder)
+ * - Create the on-disk regions (temp folder)
+ * [If something fails here: we've just some trash in temp]
+ * - Move the table from temp to the root directory
+ * [If something fails here: we've the table in place but some of the rows required
+ * present in META. (hbck needed)]
+ * - Add regions to META
+ * [If something fails here: we don't have regions assigned: table disabled]
+ * - Assign regions to Region Servers
+ * [If something fails here: we still have the table in disabled state]
+ * - Update ZooKeeper with the enabled state
+ */
+ private void handleCreateTable(String tableName) throws IOException, KeeperException {
+ Path tempdir = fileSystemManager.getTempDir();
+ FileSystem fs = fileSystemManager.getFileSystem();
+
+ // 1. Create Table Descriptor
+ FSTableDescriptors.createTableDescriptor(fs, tempdir, this.hTableDescriptor);
+ Path tempTableDir = new Path(tempdir, tableName);
+ Path tableDir = new Path(fileSystemManager.getRootDir(), tableName);
+
+ // 2. Create Regions
+ List<HRegionInfo> regionInfos = handleCreateHdfsRegions(tempdir, tableName);
+
+ // 3. Move Table temp directory to the hbase root location
+ if (!fs.rename(tempTableDir, tableDir)) {
+ throw new IOException("Unable to move table from temp=" + tempTableDir +
+ " to hbase root=" + tableDir);
}
- if (regionInfos.size() > 0) {
+
+ if (regionInfos != null && regionInfos.size() > 0) {
+ // 4. Add regions to META
MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
- }
- // 4. Trigger immediate assignment of the regions in round-robin fashion
- List<ServerName> servers = serverManager.getOnlineServersList();
- // Remove the deadNotExpired servers from the server list.
- assignmentManager.removeDeadNotExpiredServers(servers);
- try {
- this.assignmentManager.assignUserRegions(Arrays.asList(newRegions),
- servers);
- } catch (InterruptedException ie) {
- LOG.error("Caught " + ie + " during round-robin assignment");
- throw new IOException(ie);
+ // 5. Trigger immediate assignment of the regions in round-robin fashion
+ List<ServerName> servers = serverManager.getOnlineServersList();
+ // Remove the deadNotExpired servers from the server list.
+ assignmentManager.removeDeadNotExpiredServers(servers);
+ try {
+ this.assignmentManager.assignUserRegions(regionInfos, servers);
+ } catch (InterruptedException e) {
+ LOG.error("Caught " + e + " during round-robin assignment");
+ InterruptedIOException ie = new InterruptedIOException(e.getMessage());
+ ie.initCause(e);
+ throw ie;
+ }
}
- // 5. Set table enabled flag up in zk.
+ // 6. Set table enabled flag up in zk.
try {
- assignmentManager.getZKTable().
- setEnabledTable(this.hTableDescriptor.getNameAsString());
+ assignmentManager.getZKTable().setEnabledTable(tableName);
} catch (KeeperException e) {
- throw new IOException("Unable to ensure that the table will be" +
+ throw new IOException("Unable to ensure that " + tableName + " will be" +
" enabled because of a ZooKeeper issue", e);
}
}
-}
\ No newline at end of file
+
+ /**
+ * Create the on-disk structure for the table, and returns the regions info.
+ * @param tableRootDir directory where the table is being created
+ * @param tableName name of the table under construction
+ * @return the list of regions created
+ */
+ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir,
+ final String tableName)
+ throws IOException {
+ return ModifyRegionUtils.createRegions(conf, tableRootDir,
+ hTableDescriptor, newRegions, null);
+ }
+}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java Mon Mar 4 11:24:50 2013
@@ -24,10 +24,14 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
@@ -47,6 +51,7 @@ public class DeleteTableHandler extends
@Override
protected void handleTableOperation(List<HRegionInfo> regions)
throws IOException, KeeperException {
+ // 1. Wait because of region in transition
AssignmentManager am = this.masterServices.getAssignmentManager();
long waitTime = server.getConfiguration().
getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
@@ -63,23 +68,39 @@ public class DeleteTableHandler extends
waitTime + "ms) for region to leave region " +
region.getRegionNameAsString() + " in transitions");
}
- LOG.debug("Deleting region " + region.getRegionNameAsString() +
- " from META and FS");
- // Remove region from META
- MetaEditor.deleteRegion(this.server.getCatalogTracker(), region);
- // Delete region from FS
- this.masterServices.getMasterFileSystem().deleteRegion(region);
}
- // Delete table from FS
- this.masterServices.getMasterFileSystem().deleteTable(tableName);
- // Update table descriptor cache
- this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
- // If entry for this table in zk, and up in AssignmentManager, remove it.
+ // 2. Remove regions from META
+ LOG.debug("Deleting regions from META");
+ MetaEditor.deleteRegions(this.server.getCatalogTracker(), regions);
+
+ // 3. Move the table in /hbase/.tmp
+ LOG.debug("Moving table directory to a temp directory");
+ MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+ Path tempTableDir = mfs.moveTableToTemp(tableName);
+
+ try {
+ // 4. Delete regions from FS (temp directory)
+ FileSystem fs = mfs.getFileSystem();
+ for (HRegionInfo hri: regions) {
+ LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
+ HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
+ tempTableDir, new Path(tempTableDir, hri.getEncodedName()));
+ }
+
+ // 5. Delete table from FS (temp directory)
+ if (!fs.delete(tempTableDir, true)) {
+ LOG.error("Couldn't delete " + tempTableDir);
+ }
+ } finally {
+ // 6. Update table descriptor cache
+ this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
- am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+ // 7. If entry for this table in zk, and up in AssignmentManager, remove it.
+ am.getZKTable().setDeletedTable(Bytes.toString(tableName));
+ }
}
-
+
@Override
public String toString() {
String name = "UnknownServerName";
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java Mon Mar 4 11:24:50 2013
@@ -170,6 +170,7 @@ public class DisableTableHandler extends
while (!server.isStopped() && remaining > 0) {
Thread.sleep(waitingTimeForEvents);
regions = assignmentManager.getRegionsOfTable(tableName);
+ LOG.debug("Disable waiting until done; " + remaining + " ms remaining; " + regions);
if (regions.isEmpty()) break;
remaining = timeout - (System.currentTimeMillis() - startTime);
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java?rev=1452257&r1=1452256&r2=1452257&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java Mon Mar 4 11:24:50 2013
@@ -160,12 +160,14 @@ public abstract class TableEventHandler
}
/**
+ * Gets a TableDescriptor from the masterServices. Can Throw exceptions.
+ *
* @return Table descriptor for this table
* @throws TableExistsException
* @throws FileNotFoundException
* @throws IOException
*/
- HTableDescriptor getTableDescriptor()
+ public HTableDescriptor getTableDescriptor()
throws FileNotFoundException, IOException {
final String name = Bytes.toString(tableName);
HTableDescriptor htd =
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,150 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Handler to Clone a snapshot.
+ *
+ * <p>Uses {@link RestoreSnapshotHelper} to create a new table with the same
+ * content of the specified snapshot.
+ */
+@InterfaceAudience.Private
+public class CloneSnapshotHandler extends CreateTableHandler implements SnapshotSentinel {
+ private static final Log LOG = LogFactory.getLog(CloneSnapshotHandler.class);
+
+ private final static String NAME = "Master CloneSnapshotHandler";
+
+ private final SnapshotDescription snapshot;
+
+ private final ForeignExceptionDispatcher monitor;
+
+ private volatile boolean stopped = false;
+
+ public CloneSnapshotHandler(final MasterServices masterServices,
+ final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
+ throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
+ super(masterServices, masterServices.getMasterFileSystem(),
+ masterServices.getServerManager(), hTableDescriptor,
+ masterServices.getConfiguration(), null, masterServices.getCatalogTracker(),
+ masterServices.getAssignmentManager());
+
+ // Snapshot information
+ this.snapshot = snapshot;
+
+ // Monitor
+ this.monitor = new ForeignExceptionDispatcher();
+ }
+
+ /**
+ * Create the on-disk regions, using the tableRootDir provided by the CreateTableHandler.
+ * The cloned table will be created in a temp directory, and then the CreateTableHandler
+ * will be responsible to add the regions returned by this method to META and do the assignment.
+ */
+ @Override
+ protected List<HRegionInfo> handleCreateHdfsRegions(final Path tableRootDir, final String tableName)
+ throws IOException {
+ FileSystem fs = fileSystemManager.getFileSystem();
+ Path rootDir = fileSystemManager.getRootDir();
+ Path tableDir = new Path(tableRootDir, tableName);
+
+ try {
+ // 1. Execute the on-disk Clone
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
+ snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+ RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+ // Clone operation should not have stuff to restore or remove
+ Preconditions.checkArgument(!metaChanges.hasRegionsToRestore(),
+ "A clone should not have regions to restore");
+ Preconditions.checkArgument(!metaChanges.hasRegionsToRemove(),
+ "A clone should not have regions to remove");
+
+ // At this point the clone is complete. Next step is enabling the table.
+ LOG.info("Clone snapshot=" + snapshot.getName() + " on table=" + tableName + " completed!");
+
+ // 2. let the CreateTableHandler add the regions to meta
+ return metaChanges.getRegionsToAdd();
+ } catch (Exception e) {
+ String msg = "clone snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " failed";
+ LOG.error(msg, e);
+ IOException rse = new RestoreSnapshotException(msg, e, snapshot);
+
+ // these handlers aren't futures so we need to register the error here.
+ this.monitor.receive(new ForeignException(NAME, rse));
+ throw rse;
+ }
+ }
+
+ @Override
+ protected void completed(final Throwable exception) {
+ this.stopped = true;
+ }
+
+ @Override
+ public boolean isFinished() {
+ return this.stopped;
+ }
+
+ @Override
+ public SnapshotDescription getSnapshot() {
+ return snapshot;
+ }
+
+ @Override
+ public void cancel(String why) {
+ if (this.stopped) return;
+ this.stopped = true;
+ LOG.info("Stopping clone snapshot=" + snapshot + " because: " + why);
+ this.monitor.receive(new ForeignException(NAME, new CancellationException(why)));
+ }
+
+ @Override
+ public ForeignException getExceptionIfFailed() {
+ return this.monitor.getException();
+ }
+}
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
+import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Take a snapshot of a disabled table.
+ * <p>
+ * Table must exist when taking the snapshot, or results are undefined.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
+ private static final Log LOG = LogFactory.getLog(DisabledTableSnapshotHandler.class);
+ private final TimeoutExceptionInjector timeoutInjector;
+
+ /**
+ * @param snapshot descriptor of the snapshot to take
+ * @param masterServices master services provider
+ * @throws IOException on unexpected error
+ */
+ public DisabledTableSnapshotHandler(SnapshotDescription snapshot,
+ final MasterServices masterServices) throws IOException {
+ super(snapshot, masterServices);
+
+ // setup the timer
+ timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
+ }
+
+ // TODO consider parallelizing these operations since they are independent. Right now its just
+ // easier to keep them serial though
+ @Override
+ public void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regionsAndLocations) throws IOException,
+ KeeperException {
+ try {
+ timeoutInjector.start();
+
+ // 1. get all the regions hosting this table.
+
+ // extract each pair to separate lists
+ Set<String> serverNames = new HashSet<String>();
+ Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
+ for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
+ regions.add(p.getFirst());
+ serverNames.add(p.getSecond().toString());
+ }
+
+ // 2. for each region, write all the info to disk
+ LOG.info("Starting to write region info and WALs for regions for offline snapshot:"
+ + SnapshotDescriptionUtils.toString(snapshot));
+ for (HRegionInfo regionInfo : regions) {
+ // 2.1 copy the regionInfo files to the snapshot
+ Path snapshotRegionDir = TakeSnapshotUtils.getRegionSnapshotDirectory(snapshot, rootDir,
+ regionInfo.getEncodedName());
+ HRegion.writeRegioninfoOnFilesystem(regionInfo, snapshotRegionDir, fs, conf);
+ // check for error for each region
+ monitor.rethrowException();
+
+ // 2.2 for each region, copy over its recovered.edits directory
+ Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
+ new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
+ monitor.rethrowException();
+
+ // 2.3 reference all the files in the region
+ new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+ monitor.rethrowException();
+ }
+
+ // 3. write the table info to disk
+ LOG.info("Starting to copy tableinfo for offline snapshot: " +
+ SnapshotDescriptionUtils.toString(snapshot));
+ TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs,
+ FSUtils.getRootDir(conf));
+ tableInfoCopyTask.call();
+ monitor.rethrowException();
+ } catch (Exception e) {
+ // make sure we capture the exception to propagate back to the client later
+ String reason = "Failed snapshot " + SnapshotDescriptionUtils.toString(snapshot)
+ + " due to exception:" + e.getMessage();
+ ForeignException ee = new ForeignException(reason, e);
+ monitor.receive(ee);
+ } finally {
+ LOG.debug("Marking snapshot" + SnapshotDescriptionUtils.toString(snapshot)
+ + " as finished.");
+
+ // 6. mark the timer as finished - even if we got an exception, we don't need to time the
+ // operation any further
+ timeoutInjector.complete();
+ }
+ }
+}
\ No newline at end of file
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
+import org.apache.hadoop.hbase.util.Pair;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Handle the master side of taking a snapshot of an online table, regardless of snapshot type.
+ * Uses a {@link Procedure} to run the snapshot across all the involved region servers.
+ * @see ProcedureCoordinator
+ */
+@InterfaceAudience.Private
+public class EnabledTableSnapshotHandler extends TakeSnapshotHandler {
+
+ private static final Log LOG = LogFactory.getLog(EnabledTableSnapshotHandler.class);
+ private final ProcedureCoordinator coordinator;
+
+ public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master,
+ SnapshotManager manager) throws IOException {
+ super(snapshot, master);
+ this.coordinator = manager.getCoordinator();
+ }
+
+ // TODO consider switching over to using regionnames, rather than server names. This would allow
+ // regions to migrate during a snapshot, and then be involved when they are ready. Still want to
+ // enforce a snapshot time constraints, but lets us be potentially a bit more robust.
+
+ /**
+ * This method kicks off a snapshot procedure. Other than that it hangs around for various
+ * phases to complete.
+ */
+ @Override
+ protected void snapshotRegions(List<Pair<HRegionInfo, ServerName>> regions)
+ throws HBaseSnapshotException {
+ Set<String> regionServers = new HashSet<String>(regions.size());
+ for (Pair<HRegionInfo, ServerName> region : regions) {
+ regionServers.add(region.getSecond().toString());
+ }
+
+ // start the snapshot on the RS
+ Procedure proc = coordinator.startProcedure(this.monitor, this.snapshot.getName(),
+ this.snapshot.toByteArray(), Lists.newArrayList(regionServers));
+ if (proc == null) {
+ String msg = "Failed to submit distributed procedure for snapshot '"
+ + snapshot.getName() + "'";
+ LOG.error(msg);
+ throw new HBaseSnapshotException(msg);
+ }
+
+ try {
+ // wait for the snapshot to complete. A timer thread is kicked off that should cancel this
+ // if it takes too long.
+ proc.waitForCompleted();
+ LOG.info("Done waiting - snapshot for " + this.snapshot.getName() + " finished!");
+ } catch (InterruptedException e) {
+ ForeignException ee =
+ new ForeignException("Interrupted while waiting for snapshot to finish", e);
+ monitor.receive(ee);
+ Thread.currentThread().interrupt();
+ } catch (ForeignException e) {
+ monitor.receive(e);
+ }
+ }
+}
\ No newline at end of file
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+
+/**
+ * General snapshot verification on the master.
+ * <p>
+ * This is a light-weight verification mechanism for all the files in a snapshot. It doesn't
+ * attempt to verify that the files are exact copies (that would be paramount to taking the
+ * snapshot again!), but instead just attempts to ensure that the files match the expected
+ * files and are the same length.
+ * <p>
+ * Taking an online snapshots can race against other operations and this is an last line of
+ * defense. For example, if meta changes between when snapshots are taken not all regions of a
+ * table may be present. This can be caused by a region split (daughters present on this scan,
+ * but snapshot took parent), or move (snapshots only checks lists of region servers, a move could
+ * have caused a region to be skipped or done twice).
+ * <p>
+ * Current snapshot files checked:
+ * <ol>
+ * <li>SnapshotDescription is readable</li>
+ * <li>Table info is readable</li>
+ * <li>Regions</li>
+ * <ul>
+ * <li>Matching regions in the snapshot as currently in the table</li>
+ * <li>{@link HRegionInfo} matches the current and stored regions</li>
+ * <li>All referenced hfiles have valid names</li>
+ * <li>All the hfiles are present (either in .archive directory in the region)</li>
+ * <li>All recovered.edits files are present (by name) and have the correct file size</li>
+ * </ul>
+ * </ol>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class MasterSnapshotVerifier {
+
+ private SnapshotDescription snapshot;
+ private FileSystem fs;
+ private Path rootDir;
+ private String tableName;
+ private MasterServices services;
+
+ /**
+ * @param services services for the master
+ * @param snapshot snapshot to check
+ * @param rootDir root directory of the hbase installation.
+ */
+ public MasterSnapshotVerifier(MasterServices services, SnapshotDescription snapshot, Path rootDir) {
+ this.fs = services.getMasterFileSystem().getFileSystem();
+ this.services = services;
+ this.snapshot = snapshot;
+ this.rootDir = rootDir;
+ this.tableName = snapshot.getTable();
+ }
+
+ /**
+ * Verify that the snapshot in the directory is a valid snapshot
+ * @param snapshotDir snapshot directory to check
+ * @param snapshotServers {@link ServerName} of the servers that are involved in the snapshot
+ * @throws CorruptedSnapshotException if the snapshot is invalid
+ * @throws IOException if there is an unexpected connection issue to the filesystem
+ */
+ public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
+ throws CorruptedSnapshotException, IOException {
+ // verify snapshot info matches
+ verifySnapshotDescription(snapshotDir);
+
+ // check that tableinfo is a valid table description
+ verifyTableInfo(snapshotDir);
+
+ // check that each region is valid
+ verifyRegions(snapshotDir);
+ }
+
+ /**
+ * Check that the snapshot description written in the filesystem matches the current snapshot
+ * @param snapshotDir snapshot directory to check
+ */
+ private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
+ SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ if (!this.snapshot.equals(found)) {
+ throw new CorruptedSnapshotException("Snapshot read (" + found
+ + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot);
+ }
+ }
+
+ /**
+ * Check that the table descriptor for the snapshot is a valid table descriptor
+ * @param snapshotDir snapshot directory to check
+ */
+ private void verifyTableInfo(Path snapshotDir) throws IOException {
+ FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
+ }
+
+ /**
+ * Check that all the regions in the snapshot are valid, and accounted for.
+ * @param snapshotDir snapshot directory to check
+ * @throws IOException if we can't reach .META. or read the files from the FS
+ */
+ private void verifyRegions(Path snapshotDir) throws IOException {
+ List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
+ Bytes.toBytes(tableName));
+ for (HRegionInfo region : regions) {
+ // if offline split parent, skip it
+ if (region.isOffline() && (region.isSplit() || region.isSplitParent())) {
+ continue;
+ }
+
+ verifyRegion(fs, snapshotDir, region);
+ }
+ }
+
+ /**
+ * Verify that the region (regioninfo, hfiles) are valid
+ * @param fs the FileSystem instance
+ * @param snapshotDir snapshot directory to check
+ * @param region the region to check
+ */
+ private void verifyRegion(FileSystem fs, Path snapshotDir, HRegionInfo region) throws IOException {
+ // make sure we have region in the snapshot
+ Path regionDir = new Path(snapshotDir, region.getEncodedName());
+ if (!fs.exists(regionDir)) {
+ // could happen due to a move or split race.
+ throw new CorruptedSnapshotException("No region directory found for region:" + region,
+ snapshot);
+ }
+ // make sure we have the region info in the snapshot
+ Path regionInfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
+ // make sure the file exists
+ if (!fs.exists(regionInfo)) {
+ throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
+ }
+ FSDataInputStream in = fs.open(regionInfo);
+ HRegionInfo found = new HRegionInfo();
+ try {
+ found.readFields(in);
+ if (!region.equals(found)) {
+ throw new CorruptedSnapshotException("Found region info (" + found
+ + ") doesn't match expected region:" + region, snapshot);
+ }
+ } finally {
+ in.close();
+ }
+
+ // make sure we have the expected recovered edits files
+ TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
+
+ // check for the existance of each hfile
+ PathFilter familiesDirs = new FSUtils.FamilyDirFilter(fs);
+ FileStatus[] columnFamilies = FSUtils.listStatus(fs, regionDir, familiesDirs);
+ // should we do some checking here to make sure the cfs are correct?
+ if (columnFamilies == null) return;
+
+ // setup the suffixes for the snapshot directories
+ Path tableNameSuffix = new Path(tableName);
+ Path regionNameSuffix = new Path(tableNameSuffix, region.getEncodedName());
+
+ // get the potential real paths
+ Path archivedRegion = new Path(HFileArchiveUtil.getArchivePath(services.getConfiguration()),
+ regionNameSuffix);
+ Path realRegion = new Path(rootDir, regionNameSuffix);
+
+ // loop through each cf and check we can find each of the hfiles
+ for (FileStatus cf : columnFamilies) {
+ FileStatus[] hfiles = FSUtils.listStatus(fs, cf.getPath(), null);
+ // should we check if there should be hfiles?
+ if (hfiles == null || hfiles.length == 0) continue;
+
+ Path realCfDir = new Path(realRegion, cf.getPath().getName());
+ Path archivedCfDir = new Path(archivedRegion, cf.getPath().getName());
+ for (FileStatus hfile : hfiles) {
+ // make sure the name is correct
+ if (!StoreFile.validateStoreFileName(hfile.getPath().getName())) {
+ throw new CorruptedSnapshotException("HFile: " + hfile.getPath()
+ + " is not a valid hfile name.", snapshot);
+ }
+
+ // check to see if hfile is present in the real table
+ String fileName = hfile.getPath().getName();
+ Path file = new Path(realCfDir, fileName);
+ Path archived = new Path(archivedCfDir, fileName);
+ if (!fs.exists(file) && !file.equals(archived)) {
+ throw new CorruptedSnapshotException("Can't find hfile: " + hfile.getPath()
+ + " in the real (" + realCfDir + ") or archive (" + archivedCfDir
+ + ") directory for the primary table.", snapshot);
+ }
+ }
+ }
+ }
+
+ /**
+ * Check that the logs stored in the log directory for the snapshot are valid - it contains all
+ * the expected logs for all servers involved in the snapshot.
+ * @param snapshotDir snapshot directory to check
+ * @param snapshotServers list of the names of servers involved in the snapshot.
+ * @throws CorruptedSnapshotException if the hlogs in the snapshot are not correct
+ * @throws IOException if we can't reach the filesystem
+ */
+ private void verifyLogs(Path snapshotDir, Set<String> snapshotServers)
+ throws CorruptedSnapshotException, IOException {
+ Path snapshotLogDir = new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME);
+ Path logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+ TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers, snapshot,
+ snapshotLogDir);
+ }
+}
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,155 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.SnapshotSentinel;
+import org.apache.hadoop.hbase.master.handler.TableEventHandler;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Handler to Restore a snapshot.
+ *
+ * <p>Uses {@link RestoreSnapshotHelper} to replace the table content with the
+ * data available in the snapshot.
+ */
+@InterfaceAudience.Private
+public class RestoreSnapshotHandler extends TableEventHandler implements SnapshotSentinel {
+ private static final Log LOG = LogFactory.getLog(RestoreSnapshotHandler.class);
+
+ private final HTableDescriptor hTableDescriptor;
+ private final SnapshotDescription snapshot;
+
+ private final ForeignExceptionDispatcher monitor;
+ private volatile boolean stopped = false;
+
+ public RestoreSnapshotHandler(final MasterServices masterServices,
+ final SnapshotDescription snapshot, final HTableDescriptor htd)
+ throws IOException {
+ super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices);
+
+ // Snapshot information
+ this.snapshot = snapshot;
+
+ // Monitor
+ this.monitor = new ForeignExceptionDispatcher();
+
+ // Check table exists.
+ getTableDescriptor();
+
+ // This is the new schema we are going to write out as this modification.
+ this.hTableDescriptor = htd;
+ }
+
+ /**
+ * The restore table is executed in place.
+ * - The on-disk data will be restored - reference files are put in place without moving data
+ * - [if something fail here: you need to delete the table and re-run the restore]
+ * - META will be updated
+ * - [if something fail here: you need to run hbck to fix META entries]
+ * The passed in list gets changed in this method
+ */
+ @Override
+ protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
+ MasterFileSystem fileSystemManager = masterServices.getMasterFileSystem();
+ CatalogTracker catalogTracker = masterServices.getCatalogTracker();
+ FileSystem fs = fileSystemManager.getFileSystem();
+ Path rootDir = fileSystemManager.getRootDir();
+ byte[] tableName = hTableDescriptor.getName();
+ Path tableDir = HTableDescriptor.getTableDir(rootDir, tableName);
+
+ try {
+ // 1. Update descriptor
+ this.masterServices.getTableDescriptors().add(hTableDescriptor);
+
+ // 2. Execute the on-disk Restore
+ LOG.debug("Starting restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot));
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+ RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
+ masterServices.getConfiguration(), fs,
+ snapshot, snapshotDir, hTableDescriptor, tableDir, monitor);
+ RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
+
+ // 3. Applies changes to .META.
+ hris.clear();
+ if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd());
+ if (metaChanges.hasRegionsToRestore()) hris.addAll(metaChanges.getRegionsToRestore());
+ List<HRegionInfo> hrisToRemove = metaChanges.getRegionsToRemove();
+ MetaEditor.mutateRegions(catalogTracker, hrisToRemove, hris);
+
+ // At this point the restore is complete. Next step is enabling the table.
+ LOG.info("Restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot) + " on table=" +
+ Bytes.toString(tableName) + " completed!");
+ } catch (IOException e) {
+ String msg = "restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+ + " failed. Try re-running the restore command.";
+ LOG.error(msg, e);
+ monitor.receive(new ForeignException(masterServices.getServerName().toString(), e));
+ throw new RestoreSnapshotException(msg, e);
+ } finally {
+ this.stopped = true;
+ }
+ }
+
+ @Override
+ public boolean isFinished() {
+ return this.stopped;
+ }
+
+ @Override
+ public SnapshotDescription getSnapshot() {
+ return snapshot;
+ }
+
+ @Override
+ public void cancel(String why) {
+ if (this.stopped) return;
+ this.stopped = true;
+ String msg = "Stopping restore snapshot=" + SnapshotDescriptionUtils.toString(snapshot)
+ + " because: " + why;
+ LOG.info(msg);
+ CancellationException ce = new CancellationException(why);
+ this.monitor.receive(new ForeignException(masterServices.getServerName().toString(), ce));
+ }
+
+ public ForeignException getExceptionIfFailed() {
+ return this.monitor.getException();
+ }
+}
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,308 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Intelligently keep track of all the files for all the snapshots.
+ * <p>
+ * A cache of files is kept to avoid querying the {@link FileSystem} frequently. If there is a cache
+ * miss the directory modification time is used to ensure that we don't rescan directories that we
+ * already have in cache. We only check the modification times of the snapshot directories
+ * (/hbase/.snapshot/[snapshot_name]) to determine if the files need to be loaded into the cache.
+ * <p>
+ * New snapshots will be added to the cache and deleted snapshots will be removed when we refresh
+ * the cache. If the files underneath a snapshot directory are changed, but not the snapshot itself,
+ * we will ignore updates to that snapshot's files.
+ * <p>
+ * This is sufficient because each snapshot has its own directory and is added via an atomic rename
+ * <i>once</i>, when the snapshot is created. We don't need to worry about the data in the snapshot
+ * being run.
+ * <p>
+ * Further, the cache is periodically refreshed ensure that files in snapshots that were deleted are
+ * also removed from the cache.
+ * <p>
+ * A SnapshotFileInspector must be passed when creating <tt>this</tt> to allow extraction
+ * of files under the /hbase/.snapshot/[snapshot name] directory, for each snapshot.
+ * This allows you to only cache files under, for instance, all the logs in the .logs directory or
+ * all the files under all the regions.
+ * <p>
+ * <tt>this</tt> also considers all running snapshots (those under /hbase/.snapshot/.tmp) as valid
+ * snapshots and will attempt to cache files from those snapshots as well.
+ * <p>
+ * Queries about a given file are thread-safe with respect to multiple queries and cache refreshes.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotFileCache implements Stoppable {
+ interface SnapshotFileInspector {
+ /**
+ * Returns a collection of file names needed by the snapshot.
+ * @param snapshotDir {@link Path} to the snapshot directory to scan.
+ * @return the collection of file names needed by the snapshot.
+ */
+ Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException;
+ }
+
+ private static final Log LOG = LogFactory.getLog(SnapshotFileCache.class);
+ private volatile boolean stop = false;
+ private final FileSystem fs;
+ private final SnapshotFileInspector fileInspector;
+ private final Path snapshotDir;
+ private final Set<String> cache = new HashSet<String>();
+ /**
+ * This is a helper map of information about the snapshot directories so we don't need to rescan
+ * them if they haven't changed since the last time we looked.
+ */
+ private final Map<String, SnapshotDirectoryInfo> snapshots =
+ new HashMap<String, SnapshotDirectoryInfo>();
+ private final Timer refreshTimer;
+
+ private long lastModifiedTime = Long.MIN_VALUE;
+
+ /**
+ * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+ * filesystem.
+ * <p>
+ * Immediately loads the file cache.
+ * @param conf to extract the configured {@link FileSystem} where the snapshots are stored and
+ * hbase root directory
+ * @param cacheRefreshPeriod frequency (ms) with which the cache should be refreshed
+ * @param refreshThreadName name of the cache refresh thread
+ * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+ * @throws IOException if the {@link FileSystem} or root directory cannot be loaded
+ */
+ public SnapshotFileCache(Configuration conf, long cacheRefreshPeriod, String refreshThreadName,
+ SnapshotFileInspector inspectSnapshotFiles) throws IOException {
+ this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf), 0, cacheRefreshPeriod,
+ refreshThreadName, inspectSnapshotFiles);
+ }
+
+ /**
+ * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
+ * filesystem
+ * @param fs {@link FileSystem} where the snapshots are stored
+ * @param rootDir hbase root directory
+ * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
+ * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
+ * @param refreshThreadName name of the cache refresh thread
+ * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
+ */
+ public SnapshotFileCache(FileSystem fs, Path rootDir, long cacheRefreshPeriod,
+ long cacheRefreshDelay, String refreshThreadName, SnapshotFileInspector inspectSnapshotFiles) {
+ this.fs = fs;
+ this.fileInspector = inspectSnapshotFiles;
+ this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
+ // periodically refresh the file cache to make sure we aren't superfluously saving files.
+ this.refreshTimer = new Timer(refreshThreadName, true);
+ this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
+ cacheRefreshPeriod);
+ }
+
+ /**
+ * Trigger a cache refresh, even if its before the next cache refresh. Does not affect pending
+ * cache refreshes.
+ * <p>
+ * Blocks until the cache is refreshed.
+ * <p>
+ * Exposed for TESTING.
+ */
+ public void triggerCacheRefreshForTesting() {
+ try {
+ SnapshotFileCache.this.refreshCache();
+ } catch (IOException e) {
+ LOG.warn("Failed to refresh snapshot hfile cache!", e);
+ }
+ LOG.debug("Current cache:" + cache);
+ }
+
+ /**
+ * Check to see if the passed file name is contained in any of the snapshots. First checks an
+ * in-memory cache of the files to keep. If its not in the cache, then the cache is refreshed and
+ * the cache checked again for that file. This ensures that we always return <tt>true</tt> for a
+ * files that exists.
+ * <p>
+ * Note this may lead to periodic false positives for the file being referenced. Periodically, the
+ * cache is refreshed even if there are no requests to ensure that the false negatives get removed
+ * eventually. For instance, suppose you have a file in the snapshot and it gets loaded into the
+ * cache. Then at some point later that snapshot is deleted. If the cache has not been refreshed
+ * at that point, cache will still think the file system contains that file and return
+ * <tt>true</tt>, even if it is no longer present (false positive). However, if the file never was
+ * on the filesystem, we will never find it and always return <tt>false</tt>.
+ * @param fileName file to check
+ * @return <tt>false</tt> if the file is not referenced in any current or running snapshot,
+ * <tt>true</tt> if the file is in the cache.
+ * @throws IOException if there is an unexpected error reaching the filesystem.
+ */
+ // XXX this is inefficient to synchronize on the method, when what we really need to guard against
+ // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
+ // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
+ public synchronized boolean contains(String fileName) throws IOException {
+ if (this.cache.contains(fileName)) return true;
+
+ refreshCache();
+
+ // then check again
+ return this.cache.contains(fileName);
+ }
+
+ private synchronized void refreshCache() throws IOException {
+ // get the status of the snapshots directory
+ FileStatus status;
+ try {
+ status = fs.getFileStatus(snapshotDir);
+ } catch (FileNotFoundException e) {
+ LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
+ return;
+ }
+ // if the snapshot directory wasn't modified since we last check, we are done
+ if (status.getModificationTime() <= lastModifiedTime) return;
+
+ // directory was modified, so we need to reload our cache
+ // there could be a slight race here where we miss the cache, check the directory modification
+ // time, then someone updates the directory, causing us to not scan the directory again.
+ // However, snapshot directories are only created once, so this isn't an issue.
+
+ // 1. update the modified time
+ this.lastModifiedTime = status.getModificationTime();
+
+ // 2.clear the cache
+ this.cache.clear();
+ Map<String, SnapshotDirectoryInfo> known = new HashMap<String, SnapshotDirectoryInfo>();
+
+ // 3. check each of the snapshot directories
+ FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir);
+ if (snapshots == null) {
+ // remove all the remembered snapshots because we don't have any left
+ LOG.debug("No snapshots on-disk, cache empty");
+ this.snapshots.clear();
+ return;
+ }
+
+ // 3.1 iterate through the on-disk snapshots
+ for (FileStatus snapshot : snapshots) {
+ String name = snapshot.getPath().getName();
+ // its the tmp dir
+ if (name.equals(SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME)) {
+ // only add those files to the cache, but not to the known snapshots
+ FileStatus[] running = FSUtils.listStatus(fs, snapshot.getPath());
+ if (running == null) continue;
+ for (FileStatus run : running) {
+ this.cache.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+ }
+ } else {
+ SnapshotDirectoryInfo files = this.snapshots.remove(name);
+ // 3.1.1 if we don't know about the snapshot or its been modified, we need to update the files
+ // the latter could occur where I create a snapshot, then delete it, and then make a new
+ // snapshot with the same name. We will need to update the cache the information from that new
+ // snapshot, even though it has the same name as the files referenced have probably changed.
+ if (files == null || files.hasBeenModified(snapshot.getModificationTime())) {
+ // get all files for the snapshot and create a new info
+ Collection<String> storedFiles = fileInspector.filesUnderSnapshot(snapshot.getPath());
+ files = new SnapshotDirectoryInfo(snapshot.getModificationTime(), storedFiles);
+ }
+ // 3.2 add all the files to cache
+ this.cache.addAll(files.getFiles());
+ known.put(name, files);
+ }
+ }
+
+ // 4. set the snapshots we are tracking
+ this.snapshots.clear();
+ this.snapshots.putAll(known);
+ }
+
+ /**
+ * Simple helper task that just periodically attempts to refresh the cache
+ */
+ public class RefreshCacheTask extends TimerTask {
+ @Override
+ public void run() {
+ try {
+ SnapshotFileCache.this.refreshCache();
+ } catch (IOException e) {
+ LOG.warn("Failed to refresh snapshot hfile cache!", e);
+ }
+ }
+ }
+
+ @Override
+ public void stop(String why) {
+ if (!this.stop) {
+ this.stop = true;
+ this.refreshTimer.cancel();
+ }
+
+ }
+
+ @Override
+ public boolean isStopped() {
+ return this.stop;
+ }
+
+ /**
+ * Information about a snapshot directory
+ */
+ private static class SnapshotDirectoryInfo {
+ long lastModified;
+ Collection<String> files;
+
+ public SnapshotDirectoryInfo(long mtime, Collection<String> files) {
+ this.lastModified = mtime;
+ this.files = files;
+ }
+
+ /**
+ * @return the hfiles in the snapshot when <tt>this</tt> was made.
+ */
+ public Collection<String> getFiles() {
+ return this.files;
+ }
+
+ /**
+ * Check if the snapshot directory has been modified
+ * @param mtime current modification time of the directory
+ * @return <tt>true</tt> if it the modification time of the directory is newer time when we
+ * created <tt>this</tt>
+ */
+ public boolean hasBeenModified(long mtime) {
+ return this.lastModified < mtime;
+ }
+ }
+}
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a file cleaner that checks if a hfile is still used by snapshots of HBase
+ * tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
+ private static final Log LOG = LogFactory.getLog(SnapshotHFileCleaner.class);
+
+ /**
+ * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+ * snapshots (ms)
+ */
+ public static final String HFILE_CACHE_REFRESH_PERIOD_CONF_KEY =
+ "hbase.master.hfilecleaner.plugins.snapshot.period";
+
+ /** Refresh cache, by default, every 5 minutes */
+ private static final long DEFAULT_HFILE_CACHE_REFRESH_PERIOD = 300000;
+
+ /** File cache for HFiles in the completed and currently running snapshots */
+ private SnapshotFileCache cache;
+
+ @Override
+ public synchronized boolean isFileDeletable(Path filePath) {
+ try {
+ return !cache.contains(filePath.getName());
+ } catch (IOException e) {
+ LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+ e);
+ return false;
+ }
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ try {
+ long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
+ DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
+ final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+ Path rootDir = FSUtils.getRootDir(conf);
+ cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+ "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+ throws IOException {
+ return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("Failed to create cleaner util", e);
+ }
+ }
+
+ @Override
+ public void stop(String why) {
+ this.cache.stop(why);
+ }
+
+ @Override
+ public boolean isStopped() {
+ return this.cache.isStopped();
+ }
+
+ /**
+ * Exposed for Testing!
+ * @return the cache of all hfiles
+ */
+ public SnapshotFileCache getFileCacheForTesting() {
+ return this.cache;
+ }
+}
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java?rev=1452257&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotLogCleaner.java Mon Mar 4 11:24:50 2013
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.snapshot;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
+import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * Implementation of a log cleaner that checks if a log is still used by
+ * snapshots of HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SnapshotLogCleaner extends BaseLogCleanerDelegate {
+ private static final Log LOG = LogFactory.getLog(SnapshotLogCleaner.class);
+
+ /**
+ * Conf key for the frequency to attempt to refresh the cache of hfiles currently used in
+ * snapshots (ms)
+ */
+ static final String HLOG_CACHE_REFRESH_PERIOD_CONF_KEY =
+ "hbase.master.hlogcleaner.plugins.snapshot.period";
+
+ /** Refresh cache, by default, every 5 minutes */
+ private static final long DEFAULT_HLOG_CACHE_REFRESH_PERIOD = 300000;
+
+ private SnapshotFileCache cache;
+
+ @Override
+ public synchronized boolean isLogDeletable(Path filePath) {
+ try {
+ if (null == cache) return false;
+ return !cache.contains(filePath.getName());
+ } catch (IOException e) {
+ LOG.error("Exception while checking if:" + filePath + " was valid, keeping it just in case.",
+ e);
+ return false;
+ }
+ }
+
+ /**
+ * This method should only be called <b>once</b>, as it starts a thread to keep the cache
+ * up-to-date.
+ * <p>
+ * {@inheritDoc}
+ */
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ try {
+ long cacheRefreshPeriod = conf.getLong(
+ HLOG_CACHE_REFRESH_PERIOD_CONF_KEY, DEFAULT_HLOG_CACHE_REFRESH_PERIOD);
+ final FileSystem fs = FSUtils.getCurrentFileSystem(conf);
+ Path rootDir = FSUtils.getRootDir(conf);
+ cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
+ "snapshot-log-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+ public Collection<String> filesUnderSnapshot(final Path snapshotDir)
+ throws IOException {
+ return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("Failed to create snapshot log cleaner", e);
+ }
+ }
+
+ @Override
+ public void stop(String why) {
+ this.cache.stop(why);
+ }
+
+ @Override
+ public boolean isStopped() {
+ return this.cache.isStopped();
+ }
+}