You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2007/08/11 00:11:06 UTC
svn commit: r564780 [1/3] - in /lucene/hadoop/trunk/src/contrib/hbase: ./
src/java/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/
Author: jimk
Date: Fri Aug 10 15:11:05 2007
New Revision: 564780
URL: http://svn.apache.org/viewvc?view=rev&rev=564780
Log:
HADOOP-1678 On region split, master should designate which host should serve daughter splits.
Phase 1: Master balances load for new regions and when a region server fails.
Added:
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerLoad.java
Modified:
lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerInfo.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCleanRegionServerExit.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestSplit.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestToString.java
Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?view=diff&rev=564780&r1=564779&r2=564780
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Fri Aug 10 15:11:05 2007
@@ -86,3 +86,6 @@
53. HADOOP-1528 HClient for multiple tables - expose close table function
54. HADOOP-1466 Clean up warnings, visibility and javadoc issues in HBase.
55. HADOOP-1662 Make region splits faster
+ 56. HADOOP-1678 On region split, master should designate which host should
+ serve daughter splits. Phase 1: Master balances load for new regions and
+ when a region server fails.
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java?view=diff&rev=564780&r1=564779&r2=564780
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HBaseAdmin.java Fri Aug 10 15:11:05 2007
@@ -488,8 +488,9 @@
* @throws IllegalArgumentException - if the table name is reserved
*/
protected void checkReservedTableName(Text tableName) {
- if(tableName.equals(ROOT_TABLE_NAME)
- || tableName.equals(META_TABLE_NAME)) {
+ if(tableName.charAt(0) == '-' ||
+ tableName.charAt(0) == '.' ||
+ tableName.find(",") != -1) {
throw new IllegalArgumentException(tableName + " is a reserved table name");
}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java?view=diff&rev=564780&r1=564779&r2=564780
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java Fri Aug 10 15:11:05 2007
@@ -92,15 +92,17 @@
// Do we ever need to know all the information that we are storing?
/** The root table's name. */
- static final Text ROOT_TABLE_NAME = new Text("--ROOT--");
+ static final Text ROOT_TABLE_NAME = new Text("-ROOT-");
/** The META table's name. */
- static final Text META_TABLE_NAME = new Text("--META--");
+ static final Text META_TABLE_NAME = new Text(".META.");
// Defines for the column names used in both ROOT and META HBase 'meta' tables.
- /** The ROOT and META column family */
+ /** The ROOT and META column family (string) */
static final String COLUMN_FAMILY_STR = "info:";
+
+ /** The ROOT and META column family (Text) */
static final Text COLUMN_FAMILY = new Text(COLUMN_FAMILY_STR);
/** Array of meta column names */
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java?view=diff&rev=564780&r1=564779&r2=564780
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java Fri Aug 10 15:11:05 2007
@@ -28,6 +28,7 @@
import java.io.*;
import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* HLog stores all the edits to the HStore.
@@ -73,14 +74,14 @@
SequenceFile.Writer writer;
TreeMap<Long, Path> outputfiles = new TreeMap<Long, Path>();
- boolean insideCacheFlush = false;
+ volatile boolean insideCacheFlush = false;
TreeMap<Text, Long> regionToLastFlush = new TreeMap<Text, Long>();
- boolean closed = false;
- transient long logSeqNum = 0;
+ volatile boolean closed = false;
+ volatile long logSeqNum = 0;
long filenum = 0;
- transient int numEntries = 0;
+ AtomicInteger numEntries = new AtomicInteger(0);
Integer rollLock = new Integer(0);
@@ -125,7 +126,7 @@
logWriters.put(regionName, w);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Edit " + key.toString());
+ LOG.debug("Edit " + key.toString() + "=" + val.toString());
}
w.append(key, val);
}
@@ -173,6 +174,16 @@
fs.mkdirs(dir);
rollWriter();
}
+
+ synchronized void setSequenceNumber(long newvalue) {
+ if (newvalue > logSeqNum) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("changing sequence number from " + logSeqNum + " to " +
+ newvalue);
+ }
+ logSeqNum = newvalue;
+ }
+ }
/**
* Roll the log writer. That is, start writing log messages to a new file.
@@ -266,7 +277,7 @@
}
fs.delete(p);
}
- this.numEntries = 0;
+ this.numEntries.set(0);
}
}
@@ -343,13 +354,13 @@
new HLogKey(regionName, tableName, row, seqNum[counter++]);
HLogEdit logEdit = new HLogEdit(es.getKey(), es.getValue(), timestamp);
writer.append(logKey, logEdit);
- numEntries++;
+ numEntries.getAndIncrement();
}
}
/** @return How many items have been added to the log */
int getNumEntries() {
- return numEntries;
+ return numEntries.get();
}
/**
@@ -418,7 +429,7 @@
writer.append(new HLogKey(regionName, tableName, HLog.METAROW, logSeqId),
new HLogEdit(HLog.METACOLUMN, COMPLETE_CACHEFLUSH.get(),
System.currentTimeMillis()));
- numEntries++;
+ numEntries.getAndIncrement();
// Remember the most-recent flush for each region.
// This is used to delete obsolete log files.
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java?view=diff&rev=564780&r1=564779&r2=564780
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java Fri Aug 10 15:11:05 2007
@@ -72,8 +72,14 @@
/** {@inheritDoc} */
@Override
public String toString() {
- return getColumn().toString() + " " + this.getTimestamp() + " " +
- new String(getVal()).trim();
+ String value = "";
+ try {
+ value = new String(getVal(), HConstants.UTF8_ENCODING);
+
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException("UTF8 encoding not present?", e);
+ }
+ return "(" + getColumn().toString() + "/" + getTimestamp() + "/" + value + ")";
}
// Writable
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java?view=diff&rev=564780&r1=564779&r2=564780
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogKey.java Fri Aug 10 15:11:05 2007
@@ -84,7 +84,7 @@
*/
@Override
public String toString() {
- return tablename + "," + regionName + "," + row + "," + logSeqNum;
+ return tablename + "/" + regionName + "/" + row + "/" + logSeqNum;
}
/**