You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/10/15 20:18:21 UTC
svn commit: r1532464 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/wal/
test/java/org/apache/hadoop/hbase/master/
Author: liyin
Date: Tue Oct 15 18:18:20 2013
New Revision: 1532464
URL: http://svn.apache.org/r1532464
Log:
[master] Let OldLogsCleaner find out whether logs are archived to hourly directory
Author: fan
Summary: There was a static boolean initialized by HLog constructor to indicate whether logs should be archived to hourly directory. But OldLogsCleaner only runs on HMaster, which does not have HLog instance. It could not get the correct configuration so cleanup was still using the legacy way.
Test Plan:
1. Unit test passes.
2. Replaced jar on master of os 8. The cleaner has expected behavior now.
Reviewers: adela, aaiyer
Reviewed By: adela
CC: HBase Eng@
Differential Revision: https://phabricator.fb.com/D1010092
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java?rev=1532464&r1=1532463&r2=1532464&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/OldLogsCleaner.java Tue Oct 15 18:18:20 2013
@@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
@@ -47,6 +47,7 @@ public class OldLogsCleaner extends Chor
static final Log LOG = LogFactory.getLog(OldLogsCleaner.class.getName());
private final static Pattern datePattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}");
+ private static boolean ARCHIVE_TO_HOURLY_DIR = false;
// Max number we can delete on every chore, this is to make sure we don't
// issue thousands of delete commands around the same time
@@ -75,6 +76,7 @@ public class OldLogsCleaner extends Chor
conf.set("hbase.master.logcleanerplugin.impl",
"org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner");
}
+ ARCHIVE_TO_HOURLY_DIR = conf.getBoolean("hbase.hlog.archive.hourlydir", false);
this.maxDeletedLogs =
conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20);
this.fs = fs;
@@ -102,19 +104,14 @@ public class OldLogsCleaner extends Chor
* @throws IOException
*/
private void cleanHourlyDirectories(List<FileStatus> hourly) throws IOException {
- FileStatus[] files = this.fs.listStatus(this.oldLogDir);
- if (files == null || files.length == 0) {
- LOG.debug("Old log folder is empty");
- return;
- }
- Arrays.sort(files);
+ Collections.sort(hourly);
// Only delete one hourly sub-directory in one iteration. So we won't delete
// too many directories/files in a short period of time.
// When the system generates 10000-12000 log files per hour,
// around 4GB data is deleted.
- Path path = files[0].getPath();
+ Path path = hourly.get(0).getPath();
if (logCleaner.isLogDeletable(path)) {
- LOG.info("Removing old logs in " + path.toString());
+ LOG.info("Removing old logs directory " + path.toString());
this.fs.delete(path, true);
} else {
LOG.debug("Current hourly directories are not old enough. Oldest directory: " + path.toString());
@@ -169,7 +166,7 @@ public class OldLogsCleaner extends Chor
@Override
protected void chore() {
try {
- if (HLog.shouldArchiveToHourlyDir()) {
+ if (ARCHIVE_TO_HOURLY_DIR) {
FileStatus[] subdirs = this.fs.listStatus(this.oldLogDir);
List<FileStatus> hourly = new ArrayList<FileStatus>();
List<FileStatus> legacy = new ArrayList<FileStatus>();
@@ -206,4 +203,8 @@ public class OldLogsCleaner extends Chor
public static boolean isMatchDatePattern(Path file) {
return datePattern.matcher(file.getName()).matches();
}
+
+ public static boolean isOldLogsArchivedToHourlyDir() {
+ return ARCHIVE_TO_HOURLY_DIR;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java?rev=1532464&r1=1532463&r2=1532464&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java Tue Oct 15 18:18:20 2013
@@ -41,7 +41,6 @@ public class TimeToLiveLogCleaner implem
private Configuration conf;
// Configured time a log can be kept after it was closed
private long ttl;
- private boolean parseTimeFromPathName;
@Override
public boolean isLogDeletable(Path filePath) {
@@ -49,7 +48,7 @@ public class TimeToLiveLogCleaner implem
long currentTime = System.currentTimeMillis();
try {
// If the path name is in hourly format, skip getting modification time
- if (HLog.shouldArchiveToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) {
+ if (OldLogsCleaner.isOldLogsArchivedToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) {
time = HLog.DATE_FORMAT.parse(filePath.getName()).getTime();
} else {
FileStatus fStat = filePath.getFileSystem(conf).getFileStatus(filePath);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1532464&r1=1532463&r2=1532464&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 15 18:18:20 2013
@@ -1884,14 +1884,6 @@ public class HLog implements Syncable {
return new Path(oldLogsSubDir, filename);
}
- public static boolean shouldArchiveToHourlyDir() {
- return ARCHIVE_TO_HOURLY_DIR;
- }
-
- public static void setArchiveToHourlyDir(boolean b) {
- ARCHIVE_TO_HOURLY_DIR = b;
- }
-
/**
* Takes splitLogsMap and concurrently writes them to region directories using a thread pool
*
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java?rev=1532464&r1=1532463&r2=1532464&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java Tue Oct 15 18:18:20 2013
@@ -125,14 +125,13 @@ public class TestOldLogsCleaner {
// set TTL to delete 5 hours
c.setLong("hbase.master.logcleaner.ttl", 4 * 3600 * 1000);
c.setBoolean("hbase.hlog.archive.hourlydir", true);
- HLog.setArchiveToHourlyDir(true);
Path oldLogDir = new Path(TEST_UTIL.getTestDir(),
HConstants.HREGION_OLDLOGDIR_NAME);
String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8");
FileSystem fs = FileSystem.get(c);
StoppableImpl stop = new StoppableImpl();
- OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop,c, fs, oldLogDir);
+ OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop, c, fs, oldLogDir);
// Create 1 invalid directory (considering legacy logs), 10 directories representing
// recent 10 hours respectively