You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/07/22 03:11:37 UTC
svn commit: r1364203 [1/3] - in /hbase/trunk:
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-server/src/main/java/org/apache/hadoop/hbase/
hbase-server/src/main/java/org/apache/hadoop/hbase/backup/
hbase-server/src/main/java/org/apache/hadoo...
Author: larsh
Date: Sun Jul 22 01:11:36 2012
New Revision: 1364203
URL: http://svn.apache.org/viewvc?rev=1364203&view=rev
Log:
HBASE-5547 Don't delete HFiles when in "backup mode" (Jesse Yates)
Added:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveManager.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveTableMonitor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/TableHFileArchiveTracker.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/ZKTableArchiveClient.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/example/TestZooKeeperTableArchiveClient.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/CheckedArchivingHFileCleaner.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHFileArchiveUtil.java
Removed:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LogCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LogCleanerDelegate.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestLogsCleaner.java
Modified:
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
hbase/trunk/hbase-server/src/main/resources/hbase-default.xml
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1364203&r1=1364202&r2=1364203&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Sun Jul 22 01:11:36 2012
@@ -676,7 +676,7 @@ public final class HConstants {
public static final String ENABLE_WAL_COMPRESSION =
"hbase.regionserver.wal.enablecompression";
-/** Region in Transition metrics threshold time */
+ /** Region in Transition metrics threshold time */
public static final String METRICS_RIT_STUCK_WARNING_THRESHOLD="hbase.metrics.rit.stuck.warning.threshold";
public static final String LOAD_BALANCER_SLOP_KEY = "hbase.regions.slop";
@@ -689,6 +689,9 @@ public final class HConstants {
/** delimiter used between portions of a region name */
public static final int DELIMITER = ',';
+ /** Configuration key for the directory to backup HFiles for a table */
+ public static final String HFILE_ARCHIVE_DIRECTORY = "hbase.table.archive.directory";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/BaseConfigurable.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * HBase version of Hadoop's Configured class that doesn't initialize the
+ * configuration via {@link #setConf(Configuration)} in the constructor, but
+ * only sets the configuration through the {@link #setConf(Configuration)}
+ * method
+ */
+public class BaseConfigurable implements Configurable {
+
+ private Configuration conf;
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java?rev=1364203&r1=1364202&r2=1364203&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/Chore.java Sun Jul 22 01:11:36 2012
@@ -80,6 +80,7 @@ public abstract class Chore extends HasT
LOG.fatal(getName() + "error", t);
} finally {
LOG.info(getName() + " exiting");
+ cleanup();
}
}
@@ -112,4 +113,11 @@ public abstract class Chore extends HasT
protected void sleep() {
this.sleeper.sleep();
}
+
+ /**
+ * Called when the chore has completed, allowing subclasses to cleanup any
+ * extra overhead
+ */
+ protected void cleanup() {
+ }
}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,625 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.io.MultipleIOException;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+
+/**
+ * Utility class to handle the removal of HFiles (or the respective {@link StoreFile StoreFiles})
+ * for a HRegion from the {@link FileSystem}. The hfiles will be archived or deleted, depending on
+ * the state of the system.
+ */
+public class HFileArchiver {
+ private static final Log LOG = LogFactory.getLog(HFileArchiver.class);
+ private static final String SEPARATOR = ".";
+
+ private HFileArchiver() {
+ // hidden ctor since this is just a util
+ }
+
+ /**
+ * Cleans up all the files for a HRegion by archiving the HFiles to the
+ * archive directory
+ * @param fs the file system object
+ * @param info HRegionInfo for region to be deleted
+ * @throws IOException
+ */
+ public static void archiveRegion(FileSystem fs, HRegionInfo info)
+ throws IOException {
+ Path rootDir = FSUtils.getRootDir(fs.getConf());
+ archiveRegion(fs, rootDir, HTableDescriptor.getTableDir(rootDir, info.getTableName()),
+ HRegion.getRegionDir(rootDir, info));
+ }
+
+
+ /**
+ * Remove an entire region from the table directory via archiving the region's hfiles.
+ * @param fs {@link FileSystem} from which to remove the region
+ * @param rootdir {@link Path} to the root directory where hbase files are stored (for building
+ * the archive path)
+ * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
+ * @param regionDir {@link Path} to where a region is being stored (for building the archive path)
+ * @return <tt>true</tt> if the region was sucessfully deleted. <tt>false</tt> if the filesystem
+ * operations could not complete.
+ * @throws IOException if the request cannot be completed
+ */
+ public static boolean archiveRegion(FileSystem fs, Path rootdir, Path tableDir, Path regionDir)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ARCHIVING region " + regionDir.toString());
+ }
+
+ // otherwise, we archive the files
+ // make sure we can archive
+ if (tableDir == null || regionDir == null) {
+ LOG.error("No archive directory could be found because tabledir (" + tableDir
+ + ") or regiondir (" + regionDir + "was null. Deleting files instead.");
+ deleteRegionWithoutArchiving(fs, regionDir);
+ // we should have archived, but failed to. Doesn't matter if we deleted
+ // the archived files correctly or not.
+ return false;
+ }
+
+ // make sure the regiondir lives under the tabledir
+ Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString()));
+ Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(fs.getConf(), tableDir, regionDir);
+
+ LOG.debug("Have an archive directory, preparing to move files");
+ FileStatusConverter getAsFile = new FileStatusConverter(fs);
+ // otherwise, we attempt to archive the store files
+
+ // build collection of just the store directories to archive
+ Collection<File> toArchive = new ArrayList<File>();
+ final PathFilter dirFilter = new FSUtils.DirFilter(fs);
+ PathFilter nonHidden = new PathFilter() {
+ @Override
+ public boolean accept(Path file) {
+ return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
+ }
+ };
+ FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
+ // if there no files, we can just delete the directory and return;
+ if (storeDirs == null) {
+ LOG.debug("Region directory (" + regionDir + ") was empty, just deleting and returning!");
+ return deleteRegionWithoutArchiving(fs, regionDir);
+ }
+
+ // convert the files in the region to a File
+ toArchive.addAll(Lists.transform(Arrays.asList(storeDirs), getAsFile));
+ LOG.debug("Archiving:" + toArchive);
+ boolean success = false;
+ try {
+ success = resolveAndArchive(fs, regionArchiveDir, toArchive);
+ } catch (IOException e) {
+ success = false;
+ }
+
+ // if that was successful, then we delete the region
+ if (success) {
+ LOG.debug("Successfully resolved and archived, now can just delete region.");
+ return deleteRegionWithoutArchiving(fs, regionDir);
+ }
+
+ throw new IOException("Received error when attempting to archive files (" + toArchive
+ + "), cannot delete region directory.");
+ }
+
+ /**
+ * Remove the store files, either by archiving them or outright deletion
+ * @param fs the filesystem where the store files live
+ * @param parent Parent region hosting the store files
+ * @param conf {@link Configuration} to examine to determine the archive directory
+ * @param family the family hosting the store files
+ * @param compactedFiles files to be disposed of. No further reading of these files should be
+ * attempted; otherwise likely to cause an {@link IOException}
+ * @throws IOException if the files could not be correctly disposed.
+ */
+ public static void archiveStoreFiles(FileSystem fs, HRegion parent,
+ Configuration conf, byte[] family, Collection<StoreFile> compactedFiles) throws IOException {
+
+ // sometimes in testing, we don't have rss, so we need to check for that
+ if (fs == null) {
+ LOG.warn("Passed filesystem is null, so just deleting the files without archiving for region:"
+ + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family));
+ deleteStoreFilesWithoutArchiving(compactedFiles);
+ return;
+ }
+
+ // short circuit if we don't have any files to delete
+ if (compactedFiles.size() == 0) {
+ LOG.debug("No store files to dispose, done!");
+ return;
+ }
+
+ // build the archive path
+ if (parent == null || family == null) throw new IOException(
+ "Need to have a parent region and a family to archive from.");
+
+ Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family);
+
+ // make sure we don't archive if we can't and that the archive dir exists
+ if (!fs.mkdirs(storeArchiveDir)) {
+ throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
+ + Bytes.toString(family) + ", deleting compacted files instead.");
+ }
+
+ // otherwise we attempt to archive the store files
+ LOG.debug("Archiving compacted store files.");
+
+ // wrap the storefile into a File
+ StoreToFile getStorePath = new StoreToFile(fs);
+ Collection<File> storeFiles = Collections2.transform(compactedFiles, getStorePath);
+
+ // do the actual archive
+ if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) {
+ throw new IOException("Failed to archive/delete all the files for region:"
+ + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
+ + " into " + storeArchiveDir + "Something is probably arwy on the filesystem.");
+ }
+ }
+
+ /**
+ * Archive the given files and resolve any conflicts with existing files via appending the time
+ * archiving started (so all conflicts in the same group have the same timestamp appended).
+ * <p>
+ * If any of the passed files to archive are directories, archives the all files under that
+ * directory. Archive directory structure for children is the base archive directory name + the
+ * parent directory and is built recursively is passed files are directories themselves.
+ * @param fs {@link FileSystem} on which to archive the files
+ * @param baseArchiveDir base archive directory to archive the given files
+ * @param toArchive files to be archived
+ * @return <tt>true</tt> on success, <tt>false</tt> otherwise
+ * @throws IOException on unexpected failure
+ */
+ private static boolean resolveAndArchive(FileSystem fs, Path baseArchiveDir,
+ Collection<File> toArchive) throws IOException {
+ LOG.debug("Starting to archive files:" + toArchive);
+ long start = EnvironmentEdgeManager.currentTimeMillis();
+ List<File> failures = resolveAndArchive(fs, baseArchiveDir, toArchive, start);
+
+ // clean out the failures by just deleting them
+ if (failures.size() > 0) {
+ try {
+ LOG.error("Failed to complete archive, deleting extra store files.");
+ deleteFilesWithoutArchiving(failures);
+ } catch (IOException e) {
+ LOG.warn("Failed to delete store file(s) when archiving failed", e);
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Resolve any conflict with an existing archive file via timestamp-append
+ * renaming of the existing file and then archive the passed in files.
+ * @param fs {@link FileSystem} on which to archive the files
+ * @param baseArchiveDir base archive directory to store the files. If any of
+ * the files to archive are directories, will append the name of the
+ * directory to the base archive directory name, creating a parallel
+ * structure.
+ * @param toArchive files/directories that need to be archvied
+ * @param start time the archiving started - used for resolving archive
+ * conflicts.
+ * @return the list of failed to archive files.
+ * @throws IOException if an unexpected file operation exception occured
+ */
+ private static List<File> resolveAndArchive(FileSystem fs, Path baseArchiveDir,
+ Collection<File> toArchive, long start) throws IOException {
+ // short circuit if no files to move
+ if (toArchive.size() == 0) return Collections.emptyList();
+
+ LOG.debug("moving files to the archive directory: " + baseArchiveDir);
+
+ // make sure the archive directory exists
+ if (!fs.exists(baseArchiveDir)) {
+ if (!fs.mkdirs(baseArchiveDir)) {
+ throw new IOException("Failed to create the archive directory:" + baseArchiveDir
+ + ", quitting archive attempt.");
+ }
+ LOG.debug("Created archive directory:" + baseArchiveDir);
+ }
+
+ List<File> failures = new ArrayList<File>();
+ String startTime = Long.toString(start);
+ for (File file : toArchive) {
+ // if its a file archive it
+ try {
+ LOG.debug("Archiving:" + file);
+ if (file.isFile()) {
+ // attempt to archive the file
+ if (!resolveAndArchiveFile(baseArchiveDir, file, startTime)) {
+ LOG.warn("Couldn't archive " + file + " into backup directory: " + baseArchiveDir);
+ failures.add(file);
+ }
+ } else {
+ // otherwise its a directory and we need to archive all files
+ LOG.debug(file + " is a directory, archiving children files");
+ // so we add the directory name to the one base archive
+ Path parentArchiveDir = new Path(baseArchiveDir, file.getName());
+ // and then get all the files from that directory and attempt to
+ // archive those too
+ Collection<File> children = file.getChildren();
+ failures.addAll(resolveAndArchive(fs, parentArchiveDir, children, start));
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to archive file: " + file, e);
+ failures.add(file);
+ }
+ }
+ return failures;
+ }
+
+ /**
+ * Attempt to archive the passed in file to the archive directory.
+ * <p>
+ * If the same file already exists in the archive, it is moved to a timestamped directory under
+ * the archive directory and the new file is put in its place.
+ * @param archiveDir {@link Path} to the directory that stores the archives of the hfiles
+ * @param currentFile {@link Path} to the original HFile that will be archived
+ * @param archiveStartTime time the archiving started, to resolve naming conflicts
+ * @return <tt>true</tt> if the file is successfully archived. <tt>false</tt> if there was a
+ * problem, but the operation still completed.
+ * @throws IOException on failure to complete {@link FileSystem} operations.
+ */
+ private static boolean resolveAndArchiveFile(Path archiveDir, File currentFile,
+ String archiveStartTime) throws IOException {
+ // build path as it should be in the archive
+ String filename = currentFile.getName();
+ Path archiveFile = new Path(archiveDir, filename);
+ FileSystem fs = currentFile.getFileSystem();
+
+ // if the file already exists in the archive, move that one to a timestamped backup. This is a
+ // really, really unlikely situtation, where we get the same name for the existing file, but
+ // is included just for that 1 in trillion chance.
+ if (fs.exists(archiveFile)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("File:" + archiveFile + " already exists in archive, moving to "
+ + "timestamped backup and overwriting current.");
+ }
+
+ // move the archive file to the stamped backup
+ Path backedupArchiveFile = new Path(archiveDir, filename + SEPARATOR + archiveStartTime);
+ if (!fs.rename(archiveFile, backedupArchiveFile)) {
+ LOG.error("Could not rename archive file to backup: " + backedupArchiveFile
+ + ", deleting existing file in favor of newer.");
+ // try to delete the exisiting file, if we can't rename it
+ if (!fs.delete(archiveFile, false)) {
+ throw new IOException("Couldn't delete existing archive file (" + archiveFile
+ + ") or rename it to the backup file (" + backedupArchiveFile
+ + ")to make room for similarly named file.");
+ }
+ }
+ LOG.debug("Backed up archive file from: " + archiveFile);
+ }
+
+ LOG.debug("No existing file in archive for:" + archiveFile + ", free to archive original file.");
+
+ // at this point, we should have a free spot for the archive file
+ if (currentFile.moveAndClose(archiveFile)) {
+ LOG.error("Failed to archive file:" + currentFile);
+ return false;
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug("Finished archiving file from: " + currentFile + ", to: " + archiveFile);
+ }
+ return true;
+ }
+
+ /**
+ * Simple delete of regular files from the {@link FileSystem}.
+ * <p>
+ * This method is a more generic implementation that the other deleteXXX
+ * methods in this class, allowing more code reuse at the cost of a couple
+ * more, short-lived objects (which should have minimum impact on the jvm).
+ * @param fs {@link FileSystem} where the files live
+ * @param files {@link Collection} of files to be deleted
+ * @throws IOException if a file cannot be deleted. All files will be
+ * attempted to deleted before throwing the exception, rather than
+ * failing at the first file.
+ */
+ private static void deleteFilesWithoutArchiving(Collection<File> files) throws IOException {
+ List<IOException> errors = new ArrayList<IOException>(0);
+ for (File file : files) {
+ try {
+ LOG.debug("Deleting region file:" + file);
+ file.delete();
+ } catch (IOException e) {
+ LOG.error("Failed to delete file:" + file);
+ errors.add(e);
+ }
+ }
+ if (errors.size() > 0) {
+ throw MultipleIOException.createIOException(errors);
+ }
+ }
+
+ /**
+ * Without regard for backup, delete a region. Should be used with caution.
+ * @param regionDir {@link Path} to the region to be deleted.
+ * @param fs FileSystem from which to delete the region
+ * @return <tt>true</tt> on successful deletion, <tt>false</tt> otherwise
+ * @throws IOException on filesystem operation failure
+ */
+ private static boolean deleteRegionWithoutArchiving(FileSystem fs, Path regionDir)
+ throws IOException {
+ if (fs.delete(regionDir, true)) {
+ LOG.debug("Deleted all region files in: " + regionDir);
+ return true;
+ }
+ LOG.debug("Failed to delete region directory:" + regionDir);
+ return false;
+ }
+
+ /**
+ * Just do a simple delete of the given store files
+ * <p>
+ * A best effort is made to delete each of the files, rather than bailing on the first failure.
+ * <p>
+ * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes
+ * less resources, but is limited in terms of usefulness
+ * @param compactedFiles store files to delete from the file system.
+ * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
+ * throwing the exception, rather than failing at the first file.
+ */
+ private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles)
+ throws IOException {
+ LOG.debug("Deleting store files without archiving.");
+ List<IOException> errors = new ArrayList<IOException>(0);
+ for (StoreFile hsf : compactedFiles) {
+ try {
+ hsf.deleteReader();
+ } catch (IOException e) {
+ LOG.error("Failed to delete store file:" + hsf.getPath());
+ errors.add(e);
+ }
+ }
+ if (errors.size() > 0) {
+ throw MultipleIOException.createIOException(errors);
+ }
+ }
+
+ /**
+ * Adapt a type to match the {@link File} interface, which is used internally for handling
+ * archival/removal of files
+ * @param <T> type to adapt to the {@link File} interface
+ */
+ private static abstract class FileConverter<T> implements Function<T, File> {
+ protected final FileSystem fs;
+
+ public FileConverter(FileSystem fs) {
+ this.fs = fs;
+ }
+ }
+
+ /**
+ * Convert a FileStatus to something we can manage in the archiving
+ */
+ private static class FileStatusConverter extends FileConverter<FileStatus> {
+ public FileStatusConverter(FileSystem fs) {
+ super(fs);
+ }
+
+ @Override
+ public File apply(FileStatus input) {
+ return new FileablePath(fs, input.getPath());
+ }
+ }
+
+ /**
+ * Convert the {@link StoreFile} into something we can manage in the archive
+ * methods
+ */
+ private static class StoreToFile extends FileConverter<StoreFile> {
+ public StoreToFile(FileSystem fs) {
+ super(fs);
+ }
+
+ @Override
+ public File apply(StoreFile input) {
+ return new FileableStoreFile(fs, input);
+ }
+ }
+
+ /**
+ * Wrapper to handle file operations uniformly
+ */
+ private static abstract class File {
+ protected final FileSystem fs;
+
+ public File(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ /**
+ * Delete the file
+ * @throws IOException on failure
+ */
+ abstract void delete() throws IOException;
+
+ /**
+ * Check to see if this is a file or a directory
+ * @return <tt>true</tt> if it is a file, <tt>false</tt> otherwise
+ * @throws IOException on {@link FileSystem} connection error
+ */
+ abstract boolean isFile() throws IOException;
+
+ /**
+ * @return if this is a directory, returns all the children in the
+ * directory, otherwise returns an empty list
+ * @throws IOException
+ */
+ abstract Collection<File> getChildren() throws IOException;
+
+ /**
+ * close any outside readers of the file
+ * @throws IOException
+ */
+ abstract void close() throws IOException;
+
+ /**
+ * @return the name of the file (not the full fs path, just the individual
+ * file name)
+ */
+ abstract String getName();
+
+ /**
+ * @return the path to this file
+ */
+ abstract Path getPath();
+
+ /**
+ * Move the file to the given destination
+ * @param dest
+ * @return <tt>true</tt> on success
+ * @throws IOException
+ */
+ public boolean moveAndClose(Path dest) throws IOException {
+ this.close();
+ Path p = this.getPath();
+ return !fs.rename(p, dest);
+ }
+
+ /**
+ * @return the {@link FileSystem} on which this file resides
+ */
+ public FileSystem getFileSystem() {
+ return this.fs;
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass() + ", file:" + getPath().toString();
+ }
+ }
+
+ /**
+ * A {@link File} that wraps a simple {@link Path} on a {@link FileSystem}.
+ */
+ private static class FileablePath extends File {
+ private final Path file;
+ private final FileStatusConverter getAsFile;
+
+ public FileablePath(FileSystem fs, Path file) {
+ super(fs);
+ this.file = file;
+ this.getAsFile = new FileStatusConverter(fs);
+ }
+
+ @Override
+ public void delete() throws IOException {
+ if (!fs.delete(file, true)) throw new IOException("Failed to delete:" + this.file);
+ }
+
+ @Override
+ public String getName() {
+ return file.getName();
+ }
+
+ @Override
+ public Collection<File> getChildren() throws IOException {
+ if (fs.isFile(file)) return Collections.emptyList();
+ return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile);
+ }
+
+ @Override
+ public boolean isFile() throws IOException {
+ return fs.isFile(file);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // NOOP - files are implicitly closed on removal
+ }
+
+ @Override
+ Path getPath() {
+ return file;
+ }
+ }
+
+ /**
+ * {@link File} adapter for a {@link StoreFile} living on a {@link FileSystem}
+ * .
+ */
+ private static class FileableStoreFile extends File {
+ StoreFile file;
+
+ public FileableStoreFile(FileSystem fs, StoreFile store) {
+ super(fs);
+ this.file = store;
+ }
+
+ @Override
+ public void delete() throws IOException {
+ file.deleteReader();
+ }
+
+ @Override
+ public String getName() {
+ return file.getPath().getName();
+ }
+
+ @Override
+ public boolean isFile() {
+ return true;
+ }
+
+ @Override
+ public Collection<File> getChildren() throws IOException {
+ // storefiles don't have children
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void close() throws IOException {
+ file.closeReader(true);
+ }
+
+ @Override
+ Path getPath() {
+ return file.getPath();
+ }
+ }
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveManager.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveManager.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveManager.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.example;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Client-side manager for which table's hfiles should be preserved for long-term archive.
+ * @see ZKTableArchiveClient
+ * @see HFileArchiveTableMonitor
+ * @see LongTermArchivingHFileCleaner
+ */
+@InterfaceAudience.Private
+class HFileArchiveManager {
+
+ private final String archiveZnode;
+ private static final Log LOG = LogFactory.getLog(HFileArchiveManager.class);
+ private final ZooKeeperWatcher zooKeeper;
+ private volatile boolean stopped = false;
+
+ public HFileArchiveManager(HConnection connection, Configuration conf)
+ throws ZooKeeperConnectionException, IOException {
+ this.zooKeeper = new ZooKeeperWatcher(conf, "hfileArchiveManger-on-" + connection.toString(),
+ connection);
+ this.archiveZnode = ZKTableArchiveClient.getArchiveZNode(this.zooKeeper.getConfiguration(),
+ this.zooKeeper);
+ }
+
+ /**
+ * Turn on auto-backups of HFiles on the specified table.
+ * <p>
+ * When HFiles would be deleted from the hfile archive, they are instead preserved.
+ * @param table name of the table for which to preserve hfiles.
+ * @return <tt>this</tt> for chaining.
+ * @throws KeeperException if we can't reach zookeeper to update the hfile cleaner.
+ */
+ public HFileArchiveManager enableHFileBackup(byte[] table) throws KeeperException {
+ enable(this.zooKeeper, table);
+ return this;
+ }
+
+ /**
+ * Stop retaining HFiles for the given table in the archive. HFiles will be cleaned up on the next
+ * pass of the {@link HFileCleaner}, if the HFiles are retained by another cleaner.
+ * @param table name of the table for which to disable hfile retention.
+ * @return <tt>this</tt> for chaining.
+ * @throws KeeperException if if we can't reach zookeeper to update the hfile cleaner.
+ */
+ public HFileArchiveManager disableHFileBackup(byte[] table) throws KeeperException {
+ disable(this.zooKeeper, table);
+ return this;
+ }
+
+ /**
+ * Disable long-term archival of all hfiles for all tables in the cluster.
+ * @return <tt>this</tt> for chaining.
+ * @throws IOException if the number of attempts is exceeded
+ */
+ public HFileArchiveManager disableHFileBackup() throws IOException {
+ LOG.debug("Disabling backups on all tables.");
+ try {
+ ZKUtil.deleteNodeRecursively(this.zooKeeper, archiveZnode);
+ return this;
+ } catch (KeeperException e) {
+ throw new IOException("Unexpected ZK exception!", e);
+ }
+ }
+
+ /**
+ * Perform a best effort enable of hfile retention, which relies on zookeeper communicating the //
+ * * change back to the hfile cleaner.
+ * <p>
+ * No attempt is made to make sure that backups are successfully created - it is inherently an
+ * <b>asynchronous operation</b>.
+ * @param zooKeeper watcher connection to zk cluster
+ * @param table table name on which to enable archiving
+ * @throws KeeperException
+ */
+ private void enable(ZooKeeperWatcher zooKeeper, byte[] table)
+ throws KeeperException {
+ LOG.debug("Ensuring archiving znode exists");
+ ZKUtil.createAndFailSilent(zooKeeper, archiveZnode);
+
+ // then add the table to the list of znodes to archive
+ String tableNode = this.getTableNode(table);
+ LOG.debug("Creating: " + tableNode + ", data: []");
+ ZKUtil.createSetData(zooKeeper, tableNode, new byte[0]);
+ }
+
+ /**
+ * Disable all archiving of files for a given table
+ * <p>
+ * Inherently an <b>asynchronous operation</b>.
+ * @param zooKeeper watcher for the ZK cluster
+ * @param table name of the table to disable
+ * @throws KeeperException if an unexpected ZK connection issues occurs
+ */
+ private void disable(ZooKeeperWatcher zooKeeper, byte[] table) throws KeeperException {
+ // ensure the latest state of the archive node is found
+ zooKeeper.sync(archiveZnode);
+
+ // if the top-level archive node is gone, then we are done
+ if (ZKUtil.checkExists(zooKeeper, archiveZnode) < 0) {
+ return;
+ }
+ // delete the table node, from the archive
+ String tableNode = this.getTableNode(table);
+ // make sure the table is the latest version so the delete takes
+ zooKeeper.sync(tableNode);
+
+ LOG.debug("Attempting to delete table node:" + tableNode);
+ ZKUtil.deleteNodeRecursively(zooKeeper, tableNode);
+ }
+
+ public void stop() {
+ if (!this.stopped) {
+ this.stopped = true;
+ LOG.debug("Stopping HFileArchiveManager...");
+ this.zooKeeper.close();
+ }
+ }
+
+ /**
+ * Check to see if the table is currently marked for archiving
+ * @param table name of the table to check
+ * @return <tt>true</tt> if the archive znode for that table exists, <tt>false</tt> if not
+ * @throws KeeperException if an unexpected zookeeper error occurs
+ */
+ public boolean isArchivingEnabled(byte[] table) throws KeeperException {
+ String tableNode = this.getTableNode(table);
+ return ZKUtil.checkExists(zooKeeper, tableNode) >= 0;
+ }
+
+ /**
+ * Get the zookeeper node associated with archiving the given table
+ * @param table name of the table to check
+ * @return znode for the table's archive status
+ */
+ private String getTableNode(byte[] table) {
+ return ZKUtil.joinZNode(archiveZnode, Bytes.toString(table));
+ }
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveTableMonitor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveTableMonitor.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveTableMonitor.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/HFileArchiveTableMonitor.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.example;
+
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Monitor the actual tables for which HFiles are archived for long-term retention (always kept
+ * unless ZK state changes).
+ * <p>
+ * It is internally synchronized to ensure consistent view of the table state.
+ */
+public class HFileArchiveTableMonitor {
+ private static final Log LOG = LogFactory.getLog(HFileArchiveTableMonitor.class);
+ private final Set<String> archivedTables = new TreeSet<String>();
+
+ /**
+ * Set the tables to be archived. Internally adds each table and attempts to
+ * register it.
+ * <p>
+ * <b>Note: All previous tables will be removed in favor of these tables.<b>
+ * @param tables add each of the tables to be archived.
+ */
+ public synchronized void setArchiveTables(List<String> tables) {
+ archivedTables.clear();
+ archivedTables.addAll(tables);
+ }
+
+ /**
+ * Add the named table to be those being archived. Attempts to register the
+ * table
+ * @param table name of the table to be registered
+ */
+ public synchronized void addTable(String table) {
+ if (this.shouldArchiveTable(table)) {
+ LOG.debug("Already archiving table: " + table + ", ignoring it");
+ return;
+ }
+ archivedTables.add(table);
+ }
+
+ public synchronized void removeTable(String table) {
+ archivedTables.remove(table);
+ }
+
+ public synchronized void clearArchive() {
+ archivedTables.clear();
+ }
+
+ /**
+ * Determine if the given table should or should not allow its hfiles to be deleted in the archive
+ * @param tableName name of the table to check
+ * @return <tt>true</tt> if its store files should be retained, <tt>false</tt> otherwise
+ */
+ public synchronized boolean shouldArchiveTable(String tableName) {
+ return archivedTables.contains(tableName);
+ }
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/LongTermArchivingHFileCleaner.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.example;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
+import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * {@link BaseHFileCleanerDelegate} that only cleans HFiles that don't belong to a table that is
+ * currently being archived.
+ * <p>
+ * This only works properly if the {@link TimeToLiveHFileCleaner} is also enabled (it always should
+ * be), since it may take a little time for the ZK notification to propagate, in which case we may
+ * accidentally delete some files.
+ */
+@InterfaceAudience.Private
+public class LongTermArchivingHFileCleaner extends BaseHFileCleanerDelegate {
+
+ private static final Log LOG = LogFactory.getLog(LongTermArchivingHFileCleaner.class);
+
+ private TableHFileArchiveTracker archiveTracker;
+ private FileSystem fs;
+
+ @Override
+ public boolean isFileDeleteable(Path file) {
+ try {
+
+ FileStatus[] deleteStatus = FSUtils.listStatus(this.fs, file, null);
+ // if the file doesn't exist, then it can be deleted (but should never
+ // happen since deleted files shouldn't get passed in)
+ if (deleteStatus == null) return true;
+ // if its a directory with stuff in it, don't delete
+ if (deleteStatus.length > 1) return false;
+
+ // if its an empty directory, we can definitely delete
+ if (deleteStatus[0].isDir()) return true;
+
+ // otherwise, we need to check the file's table and see its being archived
+ Path family = file.getParent();
+ Path region = family.getParent();
+ Path table = region.getParent();
+
+ String tableName = table.getName();
+ return !archiveTracker.keepHFiles(tableName);
+ } catch (IOException e) {
+ LOG.error("Failed to lookup status of:" + file + ", keeping it just incase.", e);
+ return false;
+ }
+ }
+
+ @Override
+ public void setConf(Configuration config) {
+ // setup our own zookeeper connection
+ // Make my own Configuration. Then I'll have my own connection to zk that
+ // I can close myself when comes time.
+ Configuration conf = new Configuration(config);
+ super.setConf(conf);
+ try {
+ this.fs = FileSystem.get(conf);
+ this.archiveTracker = TableHFileArchiveTracker.create(conf);
+ this.archiveTracker.start();
+ } catch (KeeperException e) {
+ LOG.error("Error while configuring " + this.getClass().getName(), e);
+ } catch (IOException e) {
+ LOG.error("Error while configuring " + this.getClass().getName(), e);
+ }
+ }
+
+ @Override
+ public void stop(String reason) {
+ if (this.isStopped()) return;
+ super.stop(reason);
+ if (this.archiveTracker != null) {
+ LOG.info("Stopping " + this.archiveTracker);
+ this.archiveTracker.stop();
+ }
+
+ }
+
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/TableHFileArchiveTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/TableHFileArchiveTracker.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/TableHFileArchiveTracker.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/TableHFileArchiveTracker.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.example;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Track HFile archiving state changes in ZooKeeper. Keeps track of the tables whose HFiles should
+ * be kept in the archive.
+ * <p>
+ * {@link TableHFileArchiveTracker#start()} needs to be called to start monitoring for tables to
+ * archive.
+ */
+@InterfaceAudience.Private
+public class TableHFileArchiveTracker extends ZooKeeperListener {
+ private static final Log LOG = LogFactory.getLog(TableHFileArchiveTracker.class);
+ public static final String HFILE_ARCHIVE_ZNODE_PARENT = "hfilearchive";
+ private HFileArchiveTableMonitor monitor;
+ private String archiveHFileZNode;
+ private boolean stopped = false;
+
+ private TableHFileArchiveTracker(ZooKeeperWatcher watcher, HFileArchiveTableMonitor monitor) {
+ super(watcher);
+ watcher.registerListener(this);
+ this.monitor = monitor;
+ this.archiveHFileZNode = ZKTableArchiveClient.getArchiveZNode(watcher.getConfiguration(),
+ watcher);
+ }
+
+ /**
+ * Start monitoring for archive updates
+ * @throws KeeperException on failure to find/create nodes
+ */
+ public void start() throws KeeperException {
+ // if archiving is enabled, then read in the list of tables to archive
+ LOG.debug("Starting hfile archive tracker...");
+ this.checkEnabledAndUpdate();
+ LOG.debug("Finished starting hfile archive tracker!");
+ }
+
+ @Override
+ public void nodeCreated(String path) {
+ // if it is the archive path
+ if (!path.startsWith(archiveHFileZNode)) return;
+
+ LOG.debug("Archive node: " + path + " created");
+ // since we are already enabled, just update a single table
+ String table = path.substring(archiveHFileZNode.length());
+
+ // the top level node has come up, so read in all the tables
+ if (table.length() == 0) {
+
+ checkEnabledAndUpdate();
+ return;
+ }
+ // find the table that needs to be archived
+ try {
+ addAndReWatchTable(path);
+ } catch (KeeperException e) {
+ LOG.warn("Couldn't read zookeeper data for table for path:" + path
+ + ", not preserving a table.", e);
+ }
+ }
+
+ @Override
+ public void nodeChildrenChanged(String path) {
+ if (!path.startsWith(archiveHFileZNode)) return;
+
+ LOG.debug("Archive node: " + path + " children changed.");
+ // a table was added to the archive
+ try {
+ updateWatchedTables();
+ } catch (KeeperException e) {
+ LOG.error("Failed to update tables to archive", e);
+ }
+ }
+
+ /**
+ * Add this table to the tracker and then read a watch on that node.
+ * <p>
+ * Handles situtation where table is deleted in the time between the update and resetting the
+ * watch by deleting the table via {@link #safeStopTrackingTable(String)}
+ * @param tableZnode full zookeeper path to the table to be added
+ * @throws KeeperException if an unexpected zk exception occurs
+ */
+ private void addAndReWatchTable(String tableZnode) throws KeeperException {
+ getMonitor().addTable(ZKUtil.getNodeName(tableZnode));
+ // re-add a watch to the table created
+ // and check to make sure it wasn't deleted
+ if (!ZKUtil.watchAndCheckExists(watcher, tableZnode)) {
+ safeStopTrackingTable(tableZnode);
+ }
+ }
+
+ /**
+ * Stop tracking a table. Ensures that the table doesn't exist, but if it does, it attempts to add
+ * the table back via {@link #addAndReWatchTable(String)} - its a 'safe' removal.
+ * @param tableZnode full zookeeper path to the table to be added
+ * @throws KeeperException if an unexpected zk exception occurs
+ */
+ private void safeStopTrackingTable(String tableZnode) throws KeeperException {
+ getMonitor().removeTable(ZKUtil.getNodeName(tableZnode));
+ // if the table exists, then add and rewatch it
+ if (ZKUtil.checkExists(watcher, tableZnode) >= 0) {
+ addAndReWatchTable(tableZnode);
+ }
+ }
+
+ @Override
+ public void nodeDeleted(String path) {
+ if (!path.startsWith(archiveHFileZNode)) return;
+
+ LOG.debug("Archive node: " + path + " deleted");
+ String table = path.substring(archiveHFileZNode.length());
+ // if we stop archiving all tables
+ if (table.length() == 0) {
+ // make sure we have the tracker before deleting the archive
+ // but if we don't, we don't care about delete
+ clearTables();
+ // watches are one-time events, so we need to renew our subscription to
+ // the archive node and might as well check to make sure archiving
+ // didn't come back on at the same time
+ checkEnabledAndUpdate();
+ return;
+ }
+ // just stop archiving one table
+ // note that we don't attempt to add another watch for that table into zk.
+ // We have no assurances that the table will be archived again (or even
+ // exists for that matter), so its better not to add unnecessary load to
+ // zk for watches. If the table is created again, then we will get the
+ // notification in childrenChanaged.
+ getMonitor().removeTable(ZKUtil.getNodeName(path));
+ }
+
+ /**
+ * Sets the watch on the top-level archive znode, and then updates the montior with the current
+ * tables that should be archived (and ensures that those nodes are watched as well).
+ */
+ private void checkEnabledAndUpdate() {
+ try {
+ if (ZKUtil.watchAndCheckExists(watcher, archiveHFileZNode)) {
+ LOG.debug(archiveHFileZNode + " znode does exist, checking for tables to archive");
+
+ // update the tables we should backup, to get the most recent state.
+ // This is safer than also watching for children and then hoping we get
+ // all the updates as it makes sure we get and watch all the children
+ updateWatchedTables();
+ } else {
+ LOG.debug("Archiving not currently enabled, waiting");
+ }
+ } catch (KeeperException e) {
+ LOG.warn("Failed to watch for archiving znode", e);
+ }
+ }
+
+ /**
+ * Read the list of children under the archive znode as table names and then sets those tables to
+ * the list of tables that we should archive
+ * @throws KeeperException if there is an unexpected zk exception
+ */
+ private void updateWatchedTables() throws KeeperException {
+ // get the children and watch for new children
+ LOG.debug("Updating watches on tables to archive.");
+ // get the children and add watches for each of the children
+ List<String> tables = ZKUtil.listChildrenAndWatchThem(watcher, archiveHFileZNode);
+ LOG.debug("Starting archive for tables:" + tables);
+ // if archiving is still enabled
+ if (tables != null && tables.size() > 0) {
+ getMonitor().setArchiveTables(tables);
+ } else {
+ LOG.debug("No tables to archive.");
+ // only if we currently have a tracker, then clear the archive
+ clearTables();
+ }
+ }
+
+ /**
+ * Remove the currently archived tables.
+ * <p>
+ * Does some intelligent checking to make sure we don't prematurely create an archive tracker.
+ */
+ private void clearTables() {
+ getMonitor().clearArchive();
+ }
+
+ /**
+ * Determine if the given table should or should not allow its hfiles to be deleted
+ * @param tableName name of the table to check
+ * @return <tt>true</tt> if its store files should be retained, <tt>false</tt> otherwise
+ */
+ public boolean keepHFiles(String tableName) {
+ return getMonitor().shouldArchiveTable(tableName);
+ }
+
+ /**
+ * @return the tracker for which tables should be archived.
+ */
+ public final HFileArchiveTableMonitor getMonitor() {
+ return this.monitor;
+ }
+
+ /**
+ * Create an archive tracker for the passed in server
+ * @param conf to read for zookeeper connection information
+ * @return ZooKeeper tracker to monitor for this server if this server should archive hfiles for a
+ * given table
+ * @throws IOException If a unexpected exception occurs
+ * @throws ZooKeeperConnectionException if we can't reach zookeeper
+ */
+ public static TableHFileArchiveTracker create(Configuration conf)
+ throws ZooKeeperConnectionException, IOException {
+ ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "hfileArchiveCleaner", null);
+ return create(zkw, new HFileArchiveTableMonitor());
+ }
+
+ /**
+ * Create an archive tracker with the special passed in table monitor. Should only be used in
+ * special cases (eg. testing)
+ * @param zkw Watcher for the ZooKeeper cluster that we should track
+ * @param monitor Monitor for which tables need hfile archiving
+ * @return ZooKeeper tracker to monitor for this server if this server should archive hfiles for a
+ * given table
+ */
+ private static TableHFileArchiveTracker create(ZooKeeperWatcher zkw,
+ HFileArchiveTableMonitor monitor) {
+ return new TableHFileArchiveTracker(zkw, monitor);
+ }
+
+ public ZooKeeperWatcher getZooKeeperWatcher() {
+ return this.watcher;
+ }
+
+ /**
+ * Stop this tracker and the passed zookeeper
+ */
+ public void stop() {
+ if (this.stopped) return;
+ this.stopped = true;
+ this.watcher.close();
+ }
+}
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/ZKTableArchiveClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/ZKTableArchiveClient.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/ZKTableArchiveClient.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/example/ZKTableArchiveClient.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.example;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Example class for how to use the table archiving coordinated via zookeeper
+ */
+@InterfaceAudience.Public
+public class ZKTableArchiveClient extends Configured {
+
+ /** Configuration key for the archive node. */
+ private static final String ZOOKEEPER_ZNODE_HFILE_ARCHIVE_KEY = "zookeeper.znode.hfile.archive";
+ private HConnection connection;
+
+ public ZKTableArchiveClient(Configuration conf, HConnection connection) {
+ super(conf);
+ this.connection = connection;
+ }
+
+ /**
+ * Turn on backups for all HFiles for the given table.
+ * <p>
+ * All deleted hfiles are moved to the archive directory under the table directory, rather than
+ * being deleted.
+ * <p>
+ * If backups are already enabled for this table, does nothing.
+ * <p>
+ * If the table does not exist, the archiving the table's hfiles is still enabled as a future
+ * table with that name may be created shortly.
+ * @param table name of the table to start backing up
+ * @throws IOException if an unexpected exception occurs
+ * @throws KeeperException if zookeeper can't be reached
+ */
+ public void enableHFileBackupAsync(final byte[] table) throws IOException, KeeperException {
+ createHFileArchiveManager().enableHFileBackup(table).stop();
+ }
+
+ /**
+ * Disable hfile backups for the given table.
+ * <p>
+ * Previously backed up files are still retained (if present).
+ * <p>
+ * Asynchronous operation - some extra HFiles may be retained, in the archive directory after
+ * disable is called, dependent on the latency in zookeeper to the servers.
+ * @param table name of the table stop backing up
+ * @throws IOException if an unexpected exception occurs
+ * @throws KeeperException if zookeeper can't be reached
+ */
+ public void disableHFileBackup(String table) throws IOException, KeeperException {
+ disableHFileBackup(Bytes.toBytes(table));
+ }
+
+ /**
+ * Disable hfile backups for the given table.
+ * <p>
+ * Previously backed up files are still retained (if present).
+ * <p>
+ * Asynchronous operation - some extra HFiles may be retained, in the archive directory after
+ * disable is called, dependent on the latency in zookeeper to the servers.
+ * @param table name of the table stop backing up
+ * @throws IOException if an unexpected exception occurs
+ * @throws KeeperException if zookeeper can't be reached
+ */
+ public void disableHFileBackup(final byte[] table) throws IOException, KeeperException {
+ createHFileArchiveManager().disableHFileBackup(table).stop();
+ }
+
+ /**
+ * Disable hfile backups for all tables.
+ * <p>
+ * Previously backed up files are still retained (if present).
+ * <p>
+ * Asynchronous operation - some extra HFiles may be retained, in the archive directory after
+ * disable is called, dependent on the latency in zookeeper to the servers.
+ * @throws IOException if an unexpected exception occurs
+ * @throws KeeperException if zookeeper can't be reached
+ */
+ public void disableHFileBackup() throws IOException, KeeperException {
+ createHFileArchiveManager().disableHFileBackup().stop();
+ }
+
+ /**
+ * Determine if archiving is enabled (but not necessarily fully propagated) for a table
+ * @param table name of the table to check
+ * @return <tt>true</tt> if it is, <tt>false</tt> otherwise
+ * @throws IOException if a connection to ZooKeeper cannot be established
+ * @throws KeeperException
+ */
+ public boolean getArchivingEnabled(byte[] table) throws IOException, KeeperException {
+ HFileArchiveManager manager = createHFileArchiveManager();
+ try {
+ return manager.isArchivingEnabled(table);
+ } finally {
+ manager.stop();
+ }
+ }
+
+ /**
+ * Determine if archiving is enabled (but not necessarily fully propagated) for a table
+ * @param table name of the table to check
+ * @return <tt>true</tt> if it is, <tt>false</tt> otherwise
+ * @throws IOException if an unexpected network issue occurs
+ * @throws KeeperException if zookeeper can't be reached
+ */
+ public boolean getArchivingEnabled(String table) throws IOException, KeeperException {
+ return getArchivingEnabled(Bytes.toBytes(table));
+ }
+
+ /**
+ * @return A new {@link HFileArchiveManager} to manage which tables' hfiles should be archived
+ * rather than deleted.
+ * @throws KeeperException if we can't reach zookeeper
+ * @throws IOException if an unexpected network issue occurs
+ */
+ private synchronized HFileArchiveManager createHFileArchiveManager() throws KeeperException,
+ IOException {
+ return new HFileArchiveManager(this.connection, this.getConf());
+ }
+
+ /**
+ * @param conf conf to read for the base archive node
+ * @param zooKeeper zookeeper to used for building the full path
+ * @return get the znode for long-term archival of a table for
+ */
+ public static String getArchiveZNode(Configuration conf, ZooKeeperWatcher zooKeeper) {
+ return ZKUtil.joinZNode(zooKeeper.baseZNode, conf.get(ZOOKEEPER_ZNODE_HFILE_ARCHIVE_KEY,
+ TableHFileArchiveTracker.HFILE_ARCHIVE_ZNODE_PARENT));
+ }
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java?rev=1364203&r1=1364202&r2=1364203&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java Sun Jul 22 01:11:36 2012
@@ -41,11 +41,10 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.Bytes;
@@ -53,7 +52,6 @@ import org.apache.hadoop.hbase.util.FSUt
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
-
/**
* A janitor for the catalog tables. Scans the <code>.META.</code> catalog
* table on a period looking for unused regions to garbage collect.
@@ -253,7 +251,7 @@ class CatalogJanitor extends Chore {
if (hasNoReferences(a) && hasNoReferences(b)) {
LOG.debug("Deleting region " + parent.getRegionNameAsString() +
" because daughter splits no longer hold references");
- // wipe out daughter references from parent region
+ // wipe out daughter references from parent region in meta
removeDaughtersFromParent(parent);
// This latter regionOffline should not be necessary but is done for now
@@ -264,8 +262,7 @@ class CatalogJanitor extends Chore {
this.services.getAssignmentManager().regionOffline(parent);
}
FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
- Path rootdir = this.services.getMasterFileSystem().getRootDir();
- HRegion.deleteRegion(fs, rootdir, parent);
+ HFileArchiver.archiveRegion(fs, parent);
MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
result = true;
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1364203&r1=1364202&r2=1364203&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Sun Jul 22 01:11:36 2012
@@ -46,6 +46,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Chore;
import org.apache.hadoop.hbase.ClusterStatus;
@@ -85,6 +86,8 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.ipc.ProtocolSignature;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
+import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
@@ -104,6 +107,8 @@ import org.apache.hadoop.hbase.security.
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.InfoServer;
import org.apache.hadoop.hbase.util.Pair;
@@ -276,6 +281,7 @@ Server {
private CatalogJanitor catalogJanitorChore;
private LogCleaner logCleaner;
+ private HFileCleaner hfileCleaner;
private MasterCoprocessorHost cpHost;
private final ServerName serverName;
@@ -997,12 +1003,19 @@ Server {
// Start log cleaner thread
String n = Thread.currentThread().getName();
+ int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
this.logCleaner =
- new LogCleaner(conf.getInt("hbase.master.cleaner.interval", 60 * 1000),
+ new LogCleaner(cleanerInterval,
this, conf, getMasterFileSystem().getFileSystem(),
getMasterFileSystem().getOldLogDir());
Threads.setDaemonThreadRunning(logCleaner.getThread(), n + ".oldLogCleaner");
+ //start the hfile archive cleaner thread
+ Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
+ this.hfileCleaner = new HFileCleaner(cleanerInterval, this, conf, getMasterFileSystem()
+ .getFileSystem(), archiveDir);
+ Threads.setDaemonThreadRunning(hfileCleaner.getThread(), n + ".archivedHFileCleaner");
+
// Put up info server.
int port = this.conf.getInt(HConstants.MASTER_INFO_PORT, 60010);
if (port >= 0) {
@@ -1038,6 +1051,8 @@ Server {
this.rpcServerOpen = false;
// Clean up and close up shop
if (this.logCleaner!= null) this.logCleaner.interrupt();
+ if (this.hfileCleaner != null) this.hfileCleaner.interrupt();
+
if (this.infoServer != null) {
LOG.info("Stopping infoServer");
try {
@@ -2246,4 +2261,8 @@ Server {
MBeanUtil.registerMBean("Master", "Master", mxBeanInfo);
LOG.info("Registered HMaster MXBean");
}
+
+ public HFileCleaner getHFileCleaner() {
+ return this.hfileCleaner;
+ }
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1364203&r1=1364202&r2=1364203&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Sun Jul 22 01:11:36 2012
@@ -44,8 +44,10 @@ import org.apache.hadoop.hbase.InvalidFa
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
@@ -444,7 +446,7 @@ public class MasterFileSystem {
public void deleteRegion(HRegionInfo region) throws IOException {
- fs.delete(HRegion.getRegionDir(rootdir, region), true);
+ HFileArchiver.archiveRegion(fs, region);
}
public void deleteTable(byte[] tableName) throws IOException {
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.BaseConfigurable;
+
+/**
+ * Base class for the hfile cleaning function inside the master. By default, only the
+ * {@link TimeToLiveHFileCleaner} is called.
+ * <p>
+ * If other effects are needed, implement your own LogCleanerDelegate and add it to the
+ * configuration "hbase.master.hfilecleaner.plugins", which is a comma-separated list of fully
+ * qualified class names. The <code>HFileCleaner<code> will build the cleaner chain in
+ * order the order specified by the configuration.
+ * <p>
+ * For subclasses, setConf will be called exactly <i>once</i> before using the cleaner.
+ * <p>
+ * Since {@link BaseHFileCleanerDelegate HFileCleanerDelegates} are created in
+ * HFileCleaner by reflection, classes that implements this interface <b>must</b>
+ * provide a default constructor.
+ */
+@InterfaceAudience.Private
+public abstract class BaseHFileCleanerDelegate extends BaseConfigurable implements
+ FileCleanerDelegate {
+
+ private boolean stopped = false;
+
+ @Override
+ public void stop(String why) {
+ this.stopped = true;
+ }
+
+ @Override
+ public boolean isStopped() {
+ return this.stopped;
+ }
+}
\ No newline at end of file
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java?rev=1364203&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java Sun Jul 22 01:11:36 2012
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.BaseConfigurable;
+
+/**
+ * Base class for the log cleaning function inside the master. By default, two
+ * cleaners: <code>TimeToLiveLogCleaner</code> and
+ * <code>ReplicationLogCleaner</code> are called in order. So if other effects
+ * are needed, implement your own LogCleanerDelegate and add it to the
+ * configuration "hbase.master.logcleaner.plugins", which is a comma-separated
+ * list of fully qualified class names. LogsCleaner will add it to the chain.
+ * <p>
+ * HBase ships with LogsCleaner as the default implementation.
+ * <p>
+ * This interface extends Configurable, so setConf needs to be called once
+ * before using the cleaner. Since LogCleanerDelegates are created in
+ * LogsCleaner by reflection. Classes that implements this interface should
+ * provide a default constructor.
+ */
+@InterfaceAudience.Private
+public abstract class BaseLogCleanerDelegate extends BaseConfigurable implements FileCleanerDelegate {
+
+ @Override
+ public boolean isFileDeleteable(Path file) {
+ return isLogDeletable(file);
+ }
+
+ /**
+ * Should the master delete the log or keep it?
+ * <p>
+ * Implementing classes should override {@link #isFileDeleteable(Path)} instead.
+ * @param filePath full path to log.
+ * @return true if the log is deletable, false if not
+ */
+ @Deprecated
+ public abstract boolean isLogDeletable(Path filePath);
+}
\ No newline at end of file