You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/12/23 21:54:15 UTC
svn commit: r1425525 [2/7] - in /hbase/branches/0.94-test: ./ bin/ conf/
security/src/main/java/org/apache/hadoop/hbase/ipc/
security/src/main/java/org/apache/hadoop/hbase/security/access/
security/src/test/java/org/apache/hadoop/hbase/security/access/...
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Sun Dec 23 20:54:12 2012
@@ -494,8 +494,8 @@ public class HConnectionManager {
private final Object masterLock = new Object();
private volatile boolean closed;
private volatile boolean aborted;
+ private volatile boolean resetting;
private volatile HMasterInterface master;
- private volatile boolean masterChecked;
// ZooKeeper reference
private volatile ZooKeeperWatcher zooKeeper;
// ZooKeeper-based master address tracker
@@ -506,6 +506,8 @@ public class HConnectionManager {
private final Object metaRegionLock = new Object();
private final Object userRegionLock = new Object();
+
+ private final Object resetLock = new Object();
private final Configuration conf;
// Known region HServerAddress.toString() -> HRegionInterface
@@ -574,7 +576,7 @@ public class HConnectionManager {
HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
this.master = null;
- this.masterChecked = false;
+ this.resetting = false;
}
private synchronized void ensureZookeeperTrackers()
@@ -662,9 +664,7 @@ public class HConnectionManager {
this.master = null;
for (int tries = 0;
- !this.closed &&
- !this.masterChecked && this.master == null &&
- tries < numRetries;
+ !this.closed && this.master == null && tries < numRetries;
tries++) {
try {
@@ -703,10 +703,6 @@ public class HConnectionManager {
throw new RuntimeException("Thread was interrupted while trying to connect to master.");
}
}
- // Avoid re-checking in the future if this is a managed HConnection,
- // even if we failed to acquire a master.
- // (this is to retain the existing behavior before HBASE-5058)
- this.masterChecked = managed;
if (this.master == null) {
if (sn == null) {
@@ -1686,7 +1682,12 @@ public class HConnectionManager {
LOG.info("ZK session expired. This disconnect could have been" +
" caused by a network partition or a long-running GC pause," +
" either way it's recommended that you verify your environment.");
+ synchronized (resetLock) {
+ if (resetting) return;
+ this.resetting = true;
+ }
resetZooKeeperTrackers();
+ this.resetting = false;
}
return;
}
@@ -1756,7 +1757,6 @@ public class HConnectionManager {
HBaseRPC.stopProxy(master);
}
master = null;
- masterChecked = false;
}
if (stopProxy) {
for (HRegionInterface i : servers.values()) {
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTableInterface.java Sun Dec 23 20:54:12 2012
@@ -380,6 +380,7 @@ public interface HTableInterface extends
* @throws IOException if a remote or network exception occurs.
* @see RowLock
* @see #unlockRow
+ * @deprecated {@link RowLock} and associated operations are deprecated
*/
RowLock lockRow(byte[] row) throws IOException;
@@ -390,6 +391,7 @@ public interface HTableInterface extends
* @throws IOException if a remote or network exception occurs.
* @see RowLock
* @see #unlockRow
+ * @deprecated {@link RowLock} and associated operations are deprecated
*/
void unlockRow(RowLock rl) throws IOException;
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/HTablePool.java Sun Dec 23 20:54:12 2012
@@ -458,11 +458,17 @@ public class HTablePool implements Close
returnTable(table);
}
+ /**
+ * @deprecated {@link RowLock} and associated operations are deprecated
+ */
@Override
public RowLock lockRow(byte[] row) throws IOException {
return table.lockRow(row);
}
+ /**
+ * @deprecated {@link RowLock} and associated operations are deprecated
+ */
@Override
public void unlockRow(RowLock rl) throws IOException {
table.unlockRow(rl);
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Increment.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Increment.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Increment.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Increment.java Sun Dec 23 20:54:12 2012
@@ -73,6 +73,8 @@ public class Increment implements Row {
* At least one column must be incremented.
* @param row row key
* @param rowLock previously acquired row lock, or null
+ * @deprecated {@link RowLock} and associated operations are deprecated,
+ * use {@link #Increment(byte[])}
*/
public Increment(byte [] row, RowLock rowLock) {
this.row = row;
@@ -114,6 +116,7 @@ public class Increment implements Row {
/**
* Method for retrieving the increment's RowLock
* @return RowLock
+ * @deprecated {@link RowLock} and associated operations are deprecated
*/
public RowLock getRowLock() {
return new RowLock(this.row, this.lockId);
@@ -122,6 +125,7 @@ public class Increment implements Row {
/**
* Method for retrieving the increment's lockId
* @return lockId
+ * @deprecated {@link RowLock} and associated operations are deprecated
*/
public long getLockId() {
return this.lockId;
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Mutation.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Mutation.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Mutation.java Sun Dec 23 20:54:12 2012
@@ -164,6 +164,7 @@ public abstract class Mutation extends O
/**
* Method for retrieving the delete's RowLock
* @return RowLock
+ * @deprecated {@link RowLock} and associated operations are deprecated
*/
public RowLock getRowLock() {
return new RowLock(this.row, this.lockId);
@@ -173,6 +174,7 @@ public abstract class Mutation extends O
* Method for retrieving the delete's lock ID.
*
* @return The lock ID.
+ * @deprecated {@link RowLock} and associated operations are deprecated
*/
public long getLockId() {
return this.lockId;
@@ -191,6 +193,7 @@ public abstract class Mutation extends O
* @param clusterId
*/
public void setClusterId(UUID clusterId) {
+ if (clusterId == null) return;
byte[] val = new byte[2*Bytes.SIZEOF_LONG];
Bytes.putLong(val, 0, clusterId.getMostSignificantBits());
Bytes.putLong(val, Bytes.SIZEOF_LONG, clusterId.getLeastSignificantBits());
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Put.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Put.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Put.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Put.java Sun Dec 23 20:54:12 2012
@@ -67,6 +67,7 @@ public class Put extends Mutation
* Create a Put operation for the specified row, using an existing row lock.
* @param row row key
* @param rowLock previously acquired row lock, or null
+ * @deprecated {@link RowLock} and associated operations are deprecated, use {@link #Put(byte[])}
*/
public Put(byte [] row, RowLock rowLock) {
this(row, HConstants.LATEST_TIMESTAMP, rowLock);
@@ -87,6 +88,8 @@ public class Put extends Mutation
* @param row row key
* @param ts timestamp
* @param rowLock previously acquired row lock, or null
+ * @deprecated {@link RowLock} and associated operations are deprecated,
+ * use {@link #Put(byte[], long)}
*/
public Put(byte [] row, long ts, RowLock rowLock) {
if(row == null || row.length > HConstants.MAX_ROW_LENGTH) {
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Result.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Result.java Sun Dec 23 20:54:12 2012
@@ -96,7 +96,7 @@ public class Result implements Writable,
* @param kvs List of KeyValues
*/
public Result(List<KeyValue> kvs) {
- this(kvs.toArray(new KeyValue[0]));
+ this(kvs.toArray(new KeyValue[kvs.size()]));
}
/**
@@ -649,4 +649,15 @@ public class Result implements Writable,
}
}
}
+
+ /**
+ * Copy another Result into this one. Needed for the old Mapred framework
+ * @param other
+ */
+ public void copyFrom(Result other) {
+ this.row = other.row;
+ this.bytes = other.bytes;
+ this.familyMap = other.familyMap;
+ this.kvs = other.kvs;
+ }
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/RowLock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/RowLock.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/RowLock.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/RowLock.java Sun Dec 23 20:54:12 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.client;
/**
* Holds row name and lock id.
+ * @deprecated {@link RowLock} and associated operations are deprecated.
*/
public class RowLock {
private byte [] row = null;
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Scan.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Scan.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/client/Scan.java Sun Dec 23 20:54:12 2012
@@ -215,6 +215,9 @@ public class Scan extends OperationWithA
if(set == null) {
set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
}
+ if (qualifier == null) {
+ qualifier = HConstants.EMPTY_BYTE_ARRAY;
+ }
set.add(qualifier);
familyMap.put(family, set);
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java Sun Dec 23 20:54:12 2012
@@ -58,14 +58,14 @@ public abstract class BaseRegionObserver
public void stop(CoprocessorEnvironment e) throws IOException { }
@Override
- public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) { }
+ public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException { }
@Override
public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) { }
@Override
- public void preClose(ObserverContext<RegionCoprocessorEnvironment> e,
- boolean abortRequested) { }
+ public void preClose(ObserverContext<RegionCoprocessorEnvironment> c, boolean abortRequested)
+ throws IOException { }
@Override
public void postClose(ObserverContext<RegionCoprocessorEnvironment> e,
@@ -320,4 +320,20 @@ public abstract class BaseRegionObserver
List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException {
return hasLoaded;
}
+
+ @Override
+ public void preLockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+ byte[] row) throws IOException { }
+
+ @Override
+ public void preUnlockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+ long lockId) throws IOException { }
+
+ @Override
+ public void postLockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+ byte[] row) throws IOException { }
+
+ @Override
+ public void postUnlockRow(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] regionName,
+ long lockId) throws IOException { }
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorClassLoader.java Sun Dec 23 20:54:12 2012
@@ -65,9 +65,13 @@ public class CoprocessorClassLoader exte
"org.w3c",
"org.xml",
"sunw.",
- // Hadoop/HBase:
- "org.apache.hadoop",
+ // logging
+ "org.apache.commons.logging",
+ "org.apache.log4j",
"com.hadoop",
+ // Hadoop/HBase/ZK:
+ "org.apache.hadoop",
+ "org.apache.zookeeper",
};
/**
@@ -80,7 +84,12 @@ public class CoprocessorClassLoader exte
new Pattern[] {
Pattern.compile("^[^-]+-default\\.xml$")
};
-
+
+ /**
+ * Parent classloader used to load any class not matching the exemption list.
+ */
+ private final ClassLoader parent;
+
/**
* Creates a CoprocessorClassLoader that loads classes from the given paths.
* @param paths paths from which to load classes.
@@ -88,8 +97,12 @@ public class CoprocessorClassLoader exte
*/
public CoprocessorClassLoader(List<URL> paths, ClassLoader parent) {
super(paths.toArray(new URL[]{}), parent);
+ this.parent = parent;
+ if (parent == null) {
+ throw new IllegalArgumentException("No parent classloader!");
+ }
}
-
+
@Override
synchronized public Class<?> loadClass(String name)
throws ClassNotFoundException {
@@ -99,9 +112,9 @@ public class CoprocessorClassLoader exte
LOG.debug("Skipping exempt class " + name +
" - delegating directly to parent");
}
- return super.loadClass(name);
+ return parent.loadClass(name);
}
-
+
// Check whether the class has already been loaded:
Class<?> clasz = findLoadedClass(name);
if (clasz != null) {
@@ -123,7 +136,7 @@ public class CoprocessorClassLoader exte
LOG.debug("Class " + name + " not found - delegating to parent");
}
try {
- clasz = super.loadClass(name);
+ clasz = parent.loadClass(name);
} catch (ClassNotFoundException e2) {
// Class not found in this ClassLoader or in the parent ClassLoader
// Log some debug output before rethrowing ClassNotFoundException
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java Sun Dec 23 20:54:12 2012
@@ -20,6 +20,8 @@
package org.apache.hadoop.hbase.coprocessor;
+import com.google.common.collect.MapMaker;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -44,6 +46,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.util.*;
+import java.util.concurrent.ConcurrentMap;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
@@ -55,11 +58,13 @@ import java.util.jar.JarFile;
*/
public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
public static final String REGION_COPROCESSOR_CONF_KEY =
- "hbase.coprocessor.region.classes";
+ "hbase.coprocessor.region.classes";
+ public static final String REGIONSERVER_COPROCESSOR_CONF_KEY =
+ "hbase.coprocessor.regionserver.classes";
public static final String USER_REGION_COPROCESSOR_CONF_KEY =
- "hbase.coprocessor.user.region.classes";
+ "hbase.coprocessor.user.region.classes";
public static final String MASTER_COPROCESSOR_CONF_KEY =
- "hbase.coprocessor.master.classes";
+ "hbase.coprocessor.master.classes";
public static final String WAL_COPROCESSOR_CONF_KEY =
"hbase.coprocessor.wal.classes";
@@ -72,6 +77,15 @@ public abstract class CoprocessorHost<E
protected String pathPrefix;
protected volatile int loadSequence;
+ /*
+ * External classloaders cache keyed by external jar path.
+ * ClassLoader instance is stored as a weak-reference
+ * to allow GC'ing when no CoprocessorHost is using it
+ * (@see HBASE-7205)
+ */
+ static ConcurrentMap<Path, ClassLoader> classLoadersCache =
+ new MapMaker().concurrencyLevel(3).weakValues().makeMap();
+
public CoprocessorHost() {
pathPrefix = UUID.randomUUID().toString();
}
@@ -159,14 +173,27 @@ public abstract class CoprocessorHost<E
LOG.debug("Loading coprocessor class " + className + " with path " +
path + " and priority " + priority);
- // Have we already loaded the class, perhaps from an earlier region open
- // for the same table?
- try {
- implClass = getClass().getClassLoader().loadClass(className);
- } catch (ClassNotFoundException e) {
- LOG.info("Class " + className + " needs to be loaded from a file - " +
- path + ".");
- // go ahead to load from file system.
+ ClassLoader cl = null;
+ if (path == null) {
+ try {
+ implClass = getClass().getClassLoader().loadClass(className);
+ } catch (ClassNotFoundException e) {
+ throw new IOException("No jar path specified for " + className);
+ }
+ } else {
+ // Have we already loaded the class, perhaps from an earlier region open
+ // for the same table?
+ cl = classLoadersCache.get(path);
+ if (cl != null){
+ LOG.debug("Found classloader "+ cl + "for "+path.toString());
+ try {
+ implClass = cl.loadClass(className);
+ } catch (ClassNotFoundException e) {
+ LOG.info("Class " + className + " needs to be loaded from a file - " +
+ path + ".");
+ // go ahead to load from file system.
+ }
+ }
}
// If not, load
@@ -198,7 +225,8 @@ public abstract class CoprocessorHost<E
// unsurprisingly wants URLs, not URIs; so we will use the deprecated
// method which returns URLs for as long as it is available
List<URL> paths = new ArrayList<URL>();
- paths.add(new File(dst.toString()).getCanonicalFile().toURL());
+ URL url = new File(dst.toString()).getCanonicalFile().toURL();
+ paths.add(url);
JarFile jarFile = new JarFile(dst.toString());
Enumeration<JarEntry> entries = jarFile.entries();
@@ -215,17 +243,33 @@ public abstract class CoprocessorHost<E
}
jarFile.close();
- ClassLoader cl = new CoprocessorClassLoader(paths,
- this.getClass().getClassLoader());
- Thread.currentThread().setContextClassLoader(cl);
+ cl = new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
+ // cache cp classloader as a weak value, will be GC'ed when no reference left
+ ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
+ if (prev != null) {
+ //lost update race, use already added classloader
+ cl = prev;
+ }
+
try {
implClass = cl.loadClass(className);
} catch (ClassNotFoundException e) {
- throw new IOException(e);
+ throw new IOException("Cannot load external coprocessor class " + className, e);
}
}
- return loadInstance(implClass, priority, conf);
+ //load custom code for coprocessor
+ Thread currentThread = Thread.currentThread();
+ ClassLoader hostClassLoader = currentThread.getContextClassLoader();
+ try{
+ // switch temporarily to the thread classloader for custom CP
+ currentThread.setContextClassLoader(cl);
+ E cpInstance = loadInstance(implClass, priority, conf);
+ return cpInstance;
+ } finally {
+ // restore the fresh (host) classloader
+ currentThread.setContextClassLoader(hostClassLoader);
+ }
}
/**
@@ -302,6 +346,24 @@ public abstract class CoprocessorHost<E
}
/**
+ * Retrieves the set of classloaders used to instantiate Coprocessor classes defined in external
+ * jar files.
+ * @return A set of ClassLoader instances
+ */
+ Set<ClassLoader> getExternalClassLoaders() {
+ Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>();
+ final ClassLoader systemClassLoader = this.getClass().getClassLoader();
+ for (E env : coprocessors) {
+ ClassLoader cl = env.getInstance().getClass().getClassLoader();
+ if (cl != systemClassLoader ){
+ //do not include system classloader
+ externalClassLoaders.add(cl);
+ }
+ }
+ return externalClassLoaders;
+ }
+
+ /**
* Find a coprocessor environment by class name
* @param className the class name
* @return the coprocessor, or null if not found
@@ -475,11 +537,17 @@ public abstract class CoprocessorHost<E
return tableName;
}
+ /**
+ * @deprecated {@link RowLock} and associated operations are deprecated.
+ */
public RowLock lockRow(byte[] row) throws IOException {
throw new RuntimeException(
"row locking is not allowed within the coprocessor environment");
}
+ /**
+ * @deprecated {@link RowLock} and associated operations are deprecated.
+ */
public void unlockRow(RowLock rl) throws IOException {
throw new RuntimeException(
"row locking is not allowed within the coprocessor environment");
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java Sun Dec 23 20:54:12 2012
@@ -55,8 +55,9 @@ public interface RegionObserver extends
/**
* Called before the region is reported as open to the master.
* @param c the environment provided by the region server
+ * @throws IOException if an error occurred on the coprocessor
*/
- void preOpen(final ObserverContext<RegionCoprocessorEnvironment> c);
+ void preOpen(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
/**
* Called after the region is reported as open to the master.
@@ -227,9 +228,10 @@ public interface RegionObserver extends
* Called before the region is reported as closed to the master.
* @param c the environment provided by the region server
* @param abortRequested true if the region server is aborting
+ * @throws IOException
*/
void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
- boolean abortRequested);
+ boolean abortRequested) throws IOException;
/**
* Called after the region is reported as closed to the master.
@@ -771,4 +773,55 @@ public interface RegionObserver extends
*/
boolean postBulkLoadHFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException;
+
+ /**
+ * Called before locking a row.
+ *
+ * @param ctx
+ * @param regionName
+ * @param row
+ * @throws IOException Signals that an I/O exception has occurred.
+ * @deprecated Will be removed in 0.96
+ */
+ @Deprecated
+ void preLockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+ final byte[] regionName, final byte[] row) throws IOException;
+
+ /**
+ * Called after locking a row.
+ *
+ * @param ctx
+ * @param regionName the region name
+ * @param row
+ * @throws IOException Signals that an I/O exception has occurred.
+ * @deprecated Will be removed in 0.96
+ */
+ @Deprecated
+ void postLockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+ final byte[] regionName, final byte[] row) throws IOException;
+
+ /**
+ * Called before unlocking a row.
+ *
+ * @param ctx
+ * @param regionName
+ * @param lockId the lock id
+ * @throws IOException Signals that an I/O exception has occurred.
+ * @deprecated Will be removed in 0.96
+ */
+ @Deprecated
+ void preUnlockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+ final byte[] regionName, final long lockId) throws IOException;
+
+ /**
+ * Called after unlocking a row.
+ * @param ctx
+ * @param regionName the region name
+ * @param lockId the lock id
+ * @throws IOException Signals that an I/O exception has occurred.
+ * @deprecated Will be removed in 0.96
+ */
+ @Deprecated
+ void postUnlockRow(final ObserverContext<RegionCoprocessorEnvironment> ctx,
+ final byte[] regionName, final long lockId) throws IOException;
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Sun Dec 23 20:54:12 2012
@@ -29,6 +29,8 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1255,6 +1257,8 @@ public class HFileBlock extends SchemaCo
/** The path (if any) where this data is coming from */
protected Path path;
+ private final Lock streamLock = new ReentrantLock();
+
/** The default buffer size for our buffered streams */
public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
@@ -1329,23 +1333,9 @@ public class HFileBlock extends SchemaCo
"-byte array at offset " + destOffset);
}
- if (pread) {
- // Positional read. Better for random reads.
- int extraSize = peekIntoNextBlock ? hdrSize : 0;
-
- int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
- if (ret < size) {
- throw new IOException("Positional read of " + size + " bytes " +
- "failed at offset " + fileOffset + " (returned " + ret + ")");
- }
-
- if (ret == size || ret < size + extraSize) {
- // Could not read the next block's header, or did not try.
- return -1;
- }
- } else {
+ if (!pread && streamLock.tryLock()) {
// Seek + read. Better for scanning.
- synchronized (istream) {
+ try {
istream.seek(fileOffset);
long realOffset = istream.getPos();
@@ -1363,6 +1353,22 @@ public class HFileBlock extends SchemaCo
// Try to read the next block header.
if (!readWithExtra(istream, dest, destOffset, size, hdrSize))
return -1;
+ } finally {
+ streamLock.unlock();
+ }
+ } else {
+ // Positional read. Better for random reads; or when the streamLock is already locked.
+ int extraSize = peekIntoNextBlock ? hdrSize : 0;
+
+ int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
+ if (ret < size) {
+ throw new IOException("Positional read of " + size + " bytes " +
+ "failed at offset " + fileOffset + " (returned " + ret + ")");
+ }
+
+ if (ret == size || ret < size + extraSize) {
+ // Could not read the next block's header, or did not try.
+ return -1;
}
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java Sun Dec 23 20:54:12 2012
@@ -342,7 +342,7 @@ public class HFileBlockIndex {
midKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + keyLen);
} else {
// The middle of the root-level index.
- midKey = blockKeys[(rootCount - 1) / 2];
+ midKey = blockKeys[rootCount / 2];
}
this.midKey.set(midKey);
@@ -1429,5 +1429,4 @@ public class HFileBlockIndex {
public static int getMaxChunkSize(Configuration conf) {
return conf.getInt(MAX_CHUNK_SIZE_KEY, DEFAULT_MAX_CHUNK_SIZE);
}
-
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Sun Dec 23 20:54:12 2012
@@ -645,7 +645,7 @@ public class LruBlockCache implements Bl
// Log size
long totalSize = heapSize();
long freeSize = maxSize - totalSize;
- LruBlockCache.LOG.debug("LRU Stats: " +
+ LruBlockCache.LOG.debug("Stats: " +
"total=" + StringUtils.byteDesc(totalSize) + ", " +
"free=" + StringUtils.byteDesc(freeSize) + ", " +
"max=" + StringUtils.byteDesc(this.maxSize) + ", " +
@@ -653,11 +653,11 @@ public class LruBlockCache implements Bl
"accesses=" + stats.getRequestCount() + ", " +
"hits=" + stats.getHitCount() + ", " +
"hitRatio=" +
- (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) +
+ (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
"cachingAccesses=" + stats.getRequestCachingCount() + ", " +
"cachingHits=" + stats.getHitCachingCount() + ", " +
"cachingHitsRatio=" +
- (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) +
+ (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) + ", " +
"evictions=" + stats.getEvictionCount() + ", " +
"evicted=" + stats.getEvictedCount() + ", " +
"evictedPerRun=" + stats.evictedPerEviction());
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java Sun Dec 23 20:54:12 2012
@@ -227,7 +227,7 @@ public class TableRecordReaderImpl {
if (result != null && result.size() > 0) {
key.set(result.getRow());
lastSuccessfulRow = key.get();
- Writables.copyWritable(result, value);
+ value.copyFrom(result);
return true;
}
return false;
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java Sun Dec 23 20:54:12 2012
@@ -22,23 +22,32 @@ package org.apache.hadoop.hbase.mapreduc
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
+import java.util.UUID;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.zookeeper.KeeperException;
/**
* Import data written by {@link Export}.
@@ -47,6 +56,7 @@ public class Import {
final static String NAME = "import";
final static String CF_RENAME_PROP = "HBASE_IMPORTER_RENAME_CFS";
final static String BULK_OUTPUT_CONF_KEY = "import.bulk.output";
+ private static final Log LOG = LogFactory.getLog(Import.class);
/**
* A mapper that just writes out KeyValues.
@@ -88,6 +98,7 @@ public class Import {
static class Importer
extends TableMapper<ImmutableBytesWritable, Mutation> {
private Map<byte[], byte[]> cfRenameMap;
+ private UUID clusterId;
/**
* @param row The current table row key.
@@ -128,16 +139,32 @@ public class Import {
}
}
if (put != null) {
+ put.setClusterId(clusterId);
context.write(key, put);
}
if (delete != null) {
+ delete.setClusterId(clusterId);
context.write(key, delete);
}
}
@Override
public void setup(Context context) {
- cfRenameMap = createCfRenameMap(context.getConfiguration());
+ Configuration conf = context.getConfiguration();
+ cfRenameMap = createCfRenameMap(conf);
+ try {
+ HConnection connection = HConnectionManager.getConnection(conf);
+ ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
+ ReplicationZookeeper zkHelper = new ReplicationZookeeper(connection, conf, zkw);
+ clusterId = zkHelper.getUUIDForCluster(zkw);
+ } catch (ZooKeeperConnectionException e) {
+ LOG.error("Problem connecting to ZooKeper during task setup", e);
+ } catch (KeeperException e) {
+ LOG.error("Problem reading ZooKeeper data during task setup", e);
+ } catch (IOException e) {
+ LOG.error("Problem setting up task", e);
+ }
+
}
}
@@ -292,4 +319,4 @@ public class Import {
Job job = createSubmittableJob(conf, otherArgs);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
-}
\ No newline at end of file
+}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Sun Dec 23 20:54:12 2012
@@ -142,9 +142,10 @@ public class AssignmentManager extends Z
// store all the table names in disabling state
Set<String> disablingTables = new HashSet<String>(1);
- // store all the enabling state tablenames.
- Set<String> enablingTables = new HashSet<String>(1);
-
+ // store all the enabling state table names and corresponding online servers' regions.
+ // This may be needed to avoid calling assign twice for the regions of the ENABLING table
+ // that could have been assigned through processRIT.
+ Map<String, List<HRegionInfo>> enablingTables = new HashMap<String, List<HRegionInfo>>(1);
/**
* Server to regions assignment map.
* Contains the set of regions currently assigned to a given server.
@@ -274,6 +275,16 @@ public class AssignmentManager extends Z
}
/**
+ * Gives enabling table regions.
+ *
+ * @param tableName
+ * @return list of regionInfos
+ */
+ public List<HRegionInfo> getEnablingTableRegions(String tableName){
+ return this.enablingTables.get(tableName);
+ }
+
+ /**
* Add a regionPlan for the specified region.
* @param encodedName
* @param plan
@@ -364,7 +375,9 @@ public class AssignmentManager extends Z
// Recover the tables that were not fully moved to DISABLED state.
// These tables are in DISABLING state when the master restarted/switched.
boolean isWatcherCreated = recoverTableInDisablingState(this.disablingTables);
- recoverTableInEnablingState(this.enablingTables, isWatcherCreated);
+ recoverTableInEnablingState(this.enablingTables.keySet(), isWatcherCreated);
+ this.enablingTables.clear();
+ this.disablingTables.clear();
}
/**
@@ -509,6 +522,10 @@ public class AssignmentManager extends Z
String encodedRegionName = regionInfo.getEncodedName();
LOG.info("Processing region " + regionInfo.getRegionNameAsString() +
" in state " + data.getEventType());
+ List<HRegionInfo> hris = this.enablingTables.get(regionInfo.getTableNameAsString());
+ if (hris != null && !hris.isEmpty()) {
+ hris.remove(regionInfo);
+ }
synchronized (regionsInTransition) {
RegionState regionState = regionsInTransition.get(encodedRegionName);
if (regionState != null ||
@@ -2306,11 +2323,12 @@ public class AssignmentManager extends Z
// Skip assignment for regions of tables in DISABLING state also because
// during clean cluster startup no RS is alive and regions map also doesn't
// have any information about the regions. See HBASE-6281.
- Set<String> disablingAndDisabledTables = new HashSet<String>(this.disablingTables);
- disablingAndDisabledTables.addAll(this.zkTable.getDisabledTables());
+ Set<String> disablingDisabledAndEnablingTables = new HashSet<String>(this.disablingTables);
+ disablingDisabledAndEnablingTables.addAll(this.zkTable.getDisabledTables());
+ disablingDisabledAndEnablingTables.addAll(this.enablingTables.keySet());
// Scan META for all user regions, skipping any disabled tables
Map<HRegionInfo, ServerName> allRegions = MetaReader.fullScan(catalogTracker,
- disablingAndDisabledTables, true);
+ disablingDisabledAndEnablingTables, true);
if (allRegions == null || allRegions.isEmpty()) return;
// Get all available servers
@@ -2558,13 +2576,14 @@ public class AssignmentManager extends Z
// from ENABLED state when application calls disableTable.
// It can't be in DISABLED state, because DISABLED states transitions
// from DISABLING state.
- if (false == checkIfRegionsBelongsToEnabling(regionInfo)) {
- LOG.warn("Region " + regionInfo.getEncodedName() +
- " has null regionLocation." + " But its table " + tableName +
- " isn't in ENABLING state.");
+ boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+ addTheTablesInPartialState(regionInfo);
+ if (enabling) {
+ addToEnablingTableRegions(regionInfo);
+ } else {
+ LOG.warn("Region " + regionInfo.getEncodedName() + " has null regionLocation."
+ + " But its table " + tableName + " isn't in ENABLING state.");
}
- addTheTablesInPartialState(this.disablingTables, this.enablingTables, regionInfo,
- tableName);
} else if (!onlineServers.contains(regionLocation)) {
// Region is located on a server that isn't online
List<Pair<HRegionInfo, Result>> offlineRegions =
@@ -2575,8 +2594,7 @@ public class AssignmentManager extends Z
}
offlineRegions.add(new Pair<HRegionInfo,Result>(regionInfo, result));
disabled = checkIfRegionBelongsToDisabled(regionInfo);
- disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
- this.enablingTables, regionInfo, tableName);
+ disablingOrEnabling = addTheTablesInPartialState(regionInfo);
// need to enable the table if not disabled or disabling or enabling
// this will be used in rolling restarts
enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
@@ -2597,16 +2615,18 @@ public class AssignmentManager extends Z
}
// Region is being served and on an active server
// add only if region not in disabled and enabling table
- if (false == checkIfRegionBelongsToDisabled(regionInfo)
- && false == checkIfRegionsBelongsToEnabling(regionInfo)) {
+ boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+ disabled = checkIfRegionBelongsToDisabled(regionInfo);
+ if (!enabling && !disabled) {
synchronized (this.regions) {
regions.put(regionInfo, regionLocation);
addToServers(regionLocation, regionInfo);
}
}
- disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
- this.enablingTables, regionInfo, tableName);
- disabled = checkIfRegionBelongsToDisabled(regionInfo);
+ disablingOrEnabling = addTheTablesInPartialState(regionInfo);
+ if (enabling) {
+ addToEnablingTableRegions(regionInfo);
+ }
// need to enable the table if not disabled or disabling or enabling
// this will be used in rolling restarts
enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
@@ -2616,6 +2636,18 @@ public class AssignmentManager extends Z
return offlineServers;
}
+ private void addToEnablingTableRegions(HRegionInfo regionInfo) {
+ String tableName = regionInfo.getTableNameAsString();
+ List<HRegionInfo> hris = this.enablingTables.get(tableName);
+ if (!hris.contains(regionInfo)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding region" + regionInfo.getRegionNameAsString()
+ + " to enabling table " + tableName + ".");
+ }
+ hris.add(regionInfo);
+ }
+ }
+
private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
boolean disablingOrEnabling, String tableName) {
if (!disabled && !disablingOrEnabling
@@ -2624,14 +2656,15 @@ public class AssignmentManager extends Z
}
}
- private Boolean addTheTablesInPartialState(Set<String> disablingTables,
- Set<String> enablingTables, HRegionInfo regionInfo,
- String disablingTableName) {
+ private Boolean addTheTablesInPartialState(HRegionInfo regionInfo) {
+ String tableName = regionInfo.getTableNameAsString();
if (checkIfRegionBelongsToDisabling(regionInfo)) {
- disablingTables.add(disablingTableName);
+ this.disablingTables.add(tableName);
return true;
} else if (checkIfRegionsBelongsToEnabling(regionInfo)) {
- enablingTables.add(disablingTableName);
+ if (!this.enablingTables.containsKey(tableName)) {
+ this.enablingTables.put(tableName, new ArrayList<HRegionInfo>());
+ }
return true;
}
return false;
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Sun Dec 23 20:54:12 2012
@@ -281,6 +281,10 @@ Server {
this.rsFatals = new MemoryBoundedLogMessageBuffer(
conf.getLong("hbase.master.buffer.for.rs.fatals", 1*1024*1024));
+ // login the zookeeper client principal (if using security)
+ ZKUtil.loginClient(this.conf, "hbase.zookeeper.client.keytab.file",
+ "hbase.zookeeper.client.kerberos.principal", this.isa.getHostName());
+
// initialize server principal (if using secure Hadoop)
User.login(conf, "hbase.master.keytab.file",
"hbase.master.kerberos.principal", this.isa.getHostName());
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java Sun Dec 23 20:54:12 2012
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.ServerCommandLine;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.zookeeper.KeeperException;
public class HMasterCommandLine extends ServerCommandLine {
@@ -124,6 +125,11 @@ public class HMasterCommandLine extends
+ HConstants.ZOOKEEPER_CLIENT_PORT);
}
zooKeeperCluster.setDefaultClientPort(zkClientPort);
+
+ // login the zookeeper server principal (if using security)
+ ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
+ "hbase.zookeeper.server.kerberos.principal", null);
+
int clientPort = zooKeeperCluster.startup(zkDataPath);
if (clientPort != zkClientPort) {
String errorMsg = "Could not start ZK at requested port of " +
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Sun Dec 23 20:54:12 2012
@@ -99,6 +99,8 @@ public class MasterFileSystem {
String fsUri = this.fs.getUri().toString();
conf.set("fs.default.name", fsUri);
conf.set("fs.defaultFS", fsUri);
+ // make sure the fs has the same conf
+ fs.setConf(conf);
this.distributedLogSplitting =
conf.getBoolean("hbase.master.distributed.log.splitting", true);
if (this.distributedLogSplitting) {
@@ -454,6 +456,23 @@ public class MasterFileSystem {
// @see HRegion.checkRegioninfoOnFilesystem()
}
+ public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
+ throws IOException {
+ // archive family store files
+ Path tableDir = new Path(rootdir, region.getTableNameAsString());
+ HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
+
+ // delete the family folder
+ Path familyDir = new Path(tableDir,
+ new Path(region.getEncodedName(), Bytes.toString(familyName)));
+ if (fs.delete(familyDir, true) == false) {
+ throw new IOException("Could not delete family "
+ + Bytes.toString(familyName) + " from FileSystem for region "
+ + region.getRegionNameAsString() + "(" + region.getEncodedName()
+ + ")");
+ }
+ }
+
public void stop() {
if (splitLogManager != null) {
this.splitLogManager.stop();
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java Sun Dec 23 20:54:12 2012
@@ -21,13 +21,12 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableDescriptors;
-import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
-import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
/**
* Services Master supplies
@@ -71,6 +70,63 @@ public interface MasterServices extends
throws IOException;
/**
+ * Delete a table
+ * @param tableName The table name
+ * @throws IOException
+ */
+ public void deleteTable(final byte[] tableName) throws IOException;
+
+ /**
+ * Modify the descriptor of an existing table
+ * @param tableName The table name
+ * @param descriptor The updated table descriptor
+ * @throws IOException
+ */
+ public void modifyTable(final byte[] tableName, final HTableDescriptor descriptor)
+ throws IOException;
+
+ /**
+ * Enable an existing table
+ * @param tableName The table name
+ * @throws IOException
+ */
+ public void enableTable(final byte[] tableName) throws IOException;
+
+ /**
+ * Disable an existing table
+ * @param tableName The table name
+ * @throws IOException
+ */
+ public void disableTable(final byte[] tableName) throws IOException;
+
+ /**
+ * Add a new column to an existing table
+ * @param tableName The table name
+ * @param column The column definition
+ * @throws IOException
+ */
+ public void addColumn(final byte[] tableName, final HColumnDescriptor column)
+ throws IOException;
+
+ /**
+ * Modify the column descriptor of an existing column in an existing table
+ * @param tableName The table name
+ * @param descriptor The updated column definition
+ * @throws IOException
+ */
+ public void modifyColumn(byte[] tableName, HColumnDescriptor descriptor)
+ throws IOException;
+
+ /**
+ * Delete a column from an existing table
+ * @param tableName The table name
+ * @param columnName The column name
+ * @throws IOException
+ */
+ public void deleteColumn(final byte[] tableName, final byte[] columnName)
+ throws IOException;
+
+ /**
* @return Return table descriptors implementation.
*/
public TableDescriptors getTableDescriptors();
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Sun Dec 23 20:54:12 2012
@@ -338,7 +338,9 @@ public class SplitLogManager extends Zoo
LOG.warn("No more task remaining (ZK or task map), splitting "
+ "should have completed. Remaining tasks in ZK " + remainingInZK
+ ", active tasks in map " + actual);
- return;
+ if (remainingInZK == 0 && actual == 0) {
+ return;
+ }
}
batch.wait(100);
if (stopper.isStopped()) {
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java Sun Dec 23 20:54:12 2012
@@ -143,7 +143,9 @@ public abstract class CleanerChore<T ext
* @throws IOException if there is an unexpected filesystem error
*/
private boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
- LOG.debug("Checking directory: " + toCheck);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Checking directory: " + toCheck);
+ }
FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
// if the directory doesn't exist, then we are done
if (children == null) return true;
@@ -195,12 +197,16 @@ public abstract class CleanerChore<T ext
if (!cleaner.isFileDeletable(filePath)) {
// this file is not deletable, then we are done
- LOG.debug(filePath + " is not deletable according to:" + cleaner);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(filePath + " is not deletable according to:" + cleaner);
+ }
return false;
}
}
// delete this file if it passes all the cleaners
- LOG.debug("Removing:" + filePath + " from archive");
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Removing:" + filePath + " from archive");
+ }
boolean success = this.fs.delete(filePath, false);
if (!success) {
LOG.warn("Attempted to delete:" + filePath
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java Sun Dec 23 20:54:12 2012
@@ -65,7 +65,10 @@ public class TimeToLiveHFileCleaner exte
return false;
}
long life = currentTime - time;
- LOG.debug("Life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: " + time);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: "
+ + time);
+ }
if (life < 0) {
LOG.warn("Found a log (" + filePath + ") newer than current time (" + currentTime + " < "
+ time + "), probably a clock skew");
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java Sun Dec 23 20:54:12 2012
@@ -142,16 +142,12 @@ public class CreateTableHandler extends
List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
final int batchSize =
this.conf.getInt("hbase.master.createtable.batchsize", 100);
- HLog hlog = null;
for (int regionIdx = 0; regionIdx < this.newRegions.length; regionIdx++) {
HRegionInfo newRegion = this.newRegions[regionIdx];
// 1. Create HRegion
HRegion region = HRegion.createHRegion(newRegion,
this.fileSystemManager.getRootDir(), this.conf,
- this.hTableDescriptor, hlog);
- if (hlog == null) {
- hlog = region.getLog();
- }
+ this.hTableDescriptor, null, false, true);
regionInfos.add(region.getRegionInfo());
if (regionIdx % batchSize == 0) {
@@ -163,7 +159,6 @@ public class CreateTableHandler extends
// 3. Close the new region to flush to disk. Close log file too.
region.close();
}
- hlog.closeAndDelete();
if (regionInfos.size() > 0) {
MetaEditor.addRegionsToMeta(this.catalogTracker, regionInfos);
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java Sun Dec 23 20:54:12 2012
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.master.handler;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -27,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
@@ -34,7 +36,11 @@ import org.apache.hadoop.hbase.catalog.M
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.BulkAssigner;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
/**
@@ -46,6 +52,7 @@ public class EnableTableHandler extends
private final String tableNameStr;
private final AssignmentManager assignmentManager;
private final CatalogTracker ct;
+ private boolean retainAssignment = false;
public EnableTableHandler(Server server, byte [] tableName,
CatalogTracker catalogTracker, AssignmentManager assignmentManager,
@@ -56,6 +63,7 @@ public class EnableTableHandler extends
this.tableNameStr = Bytes.toString(tableName);
this.ct = catalogTracker;
this.assignmentManager = assignmentManager;
+ this.retainAssignment = skipTableStateCheck;
// Check if table exists
if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
throw new TableNotFoundException(Bytes.toString(tableName));
@@ -99,10 +107,12 @@ public class EnableTableHandler extends
LOG.error("Error trying to enable the table " + this.tableNameStr, e);
} catch (KeeperException e) {
LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+ } catch (InterruptedException e) {
+ LOG.error("Error trying to enable the table " + this.tableNameStr, e);
}
}
- private void handleEnableTable() throws IOException, KeeperException {
+ private void handleEnableTable() throws IOException, KeeperException, InterruptedException {
// I could check table is disabling and if so, not enable but require
// that user first finish disabling but that might be obnoxious.
@@ -111,18 +121,18 @@ public class EnableTableHandler extends
boolean done = false;
// Get the regions of this table. We're done when all listed
// tables are onlined.
- List<HRegionInfo> regionsInMeta;
- regionsInMeta = MetaReader.getTableRegions(this.ct, tableName, true);
- int countOfRegionsInTable = regionsInMeta.size();
- List<HRegionInfo> regions = regionsToAssign(regionsInMeta);
+ List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations = MetaReader
+ .getTableRegionsAndLocations(this.ct, tableName, true);
+ int countOfRegionsInTable = tableRegionsAndLocations.size();
+ List<HRegionInfo> regions = regionsToAssignWithServerName(tableRegionsAndLocations);
int regionsCount = regions.size();
if (regionsCount == 0) {
done = true;
}
LOG.info("Table has " + countOfRegionsInTable + " regions of which " +
regionsCount + " are offline.");
- BulkEnabler bd = new BulkEnabler(this.server, regions,
- countOfRegionsInTable);
+ BulkEnabler bd = new BulkEnabler(this.server, regions, countOfRegionsInTable,
+ this.retainAssignment);
try {
if (bd.bulkAssign()) {
done = true;
@@ -140,17 +150,34 @@ public class EnableTableHandler extends
/**
* @param regionsInMeta This datastructure is edited by this method.
- * @return The <code>regionsInMeta</code> list minus the regions that have
- * been onlined; i.e. List of regions that need onlining.
+ * @return List of regions neither in transition nor assigned.
* @throws IOException
*/
- private List<HRegionInfo> regionsToAssign(
- final List<HRegionInfo> regionsInMeta)
- throws IOException {
- final List<HRegionInfo> onlineRegions =
- this.assignmentManager.getRegionsOfTable(tableName);
- regionsInMeta.removeAll(onlineRegions);
- return regionsInMeta;
+ private List<HRegionInfo> regionsToAssignWithServerName(
+ final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException {
+ ServerManager serverManager = ((HMaster) this.server).getServerManager();
+ List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+ List<HRegionInfo> enablingTableRegions = this.assignmentManager
+ .getEnablingTableRegions(this.tableNameStr);
+ final List<HRegionInfo> onlineRegions = this.assignmentManager.getRegionsOfTable(tableName);
+ for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
+ HRegionInfo hri = regionLocation.getFirst();
+ ServerName sn = regionLocation.getSecond();
+ if (this.retainAssignment) {
+ // Region may be available in enablingTableRegions during master startup only.
+ if (enablingTableRegions != null && enablingTableRegions.contains(hri)) {
+ regions.add(hri);
+ if (sn != null && serverManager.isServerOnline(sn)) {
+ this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn));
+ }
+ }
+ } else if (onlineRegions.contains(hri)) {
+ continue;
+ } else {
+ regions.add(hri);
+ }
+ }
+ return regions;
}
/**
@@ -160,12 +187,14 @@ public class EnableTableHandler extends
private final List<HRegionInfo> regions;
// Count of regions in table at time this assign was launched.
private final int countOfRegionsInTable;
+ private final boolean retainAssignment;
BulkEnabler(final Server server, final List<HRegionInfo> regions,
- final int countOfRegionsInTable) {
+ final int countOfRegionsInTable,final boolean retainAssignment) {
super(server);
this.regions = regions;
this.countOfRegionsInTable = countOfRegionsInTable;
+ this.retainAssignment = retainAssignment;
}
@Override
@@ -173,7 +202,7 @@ public class EnableTableHandler extends
boolean roundRobinAssignment = this.server.getConfiguration().getBoolean(
"hbase.master.enabletable.roundrobin", false);
- if (!roundRobinAssignment) {
+ if (retainAssignment || !roundRobinAssignment) {
for (HRegionInfo region : regions) {
if (assignmentManager.isRegionInTransition(region) != null) {
continue;
@@ -181,7 +210,11 @@ public class EnableTableHandler extends
final HRegionInfo hri = region;
pool.execute(new Runnable() {
public void run() {
- assignmentManager.assign(hri, true);
+ if (retainAssignment) {
+ assignmentManager.assign(hri, true, false, false);
+ } else {
+ assignmentManager.assign(hri, true);
+ }
}
});
}
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java?rev=1425525&r1=1425524&r2=1425525&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/master/handler/TableDeleteFamilyHandler.java Sun Dec 23 20:54:12 2012
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -49,6 +50,12 @@ public class TableDeleteFamilyHandler ex
this.masterServices.getMasterFileSystem().deleteColumn(tableName, familyName);
// Update in-memory descriptor cache
this.masterServices.getTableDescriptors().add(htd);
+ // Remove the column family from the file system
+ MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
+ for (HRegionInfo hri : hris) {
+ // Delete the family directory in FS for all the regions one by one
+ mfs.deleteFamilyFromFS(hri, familyName);
+ }
}
@Override