You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/04/06 18:35:31 UTC
svn commit: r1310457 [1/3] - in /accumulo/trunk:
core/src/main/java/org/apache/accumulo/core/client/impl/
core/src/main/java/org/apache/accumulo/core/conf/
core/src/main/java/org/apache/accumulo/core/data/
server/src/main/java/org/apache/accumulo/serve...
Author: ecn
Date: Fri Apr 6 16:35:29 2012
New Revision: 1310457
URL: http://svn.apache.org/viewvc?rev=1310457&view=rev
Log:
ACCUMULO-14: got rid of ServerConfiguration.getInstance()
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletType.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ServerConfiguration.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfWatcher.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tserverOps/StopLogger.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/LogService.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/ZooKeeperStatus.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TServersServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/trace/Basic.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/security/SecurityConstants.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/FileManager.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/InMemoryMap.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/MemoryManager.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LoggerStrategy.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/RemoteLogger.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/WrongTabletTest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/BulkFileTest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/PermissionsTest.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/scan/CollectTabletStats.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/CheckForMetadataProblems.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/DumpTabletsOnServer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/ListInstances.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/OfflineMetadataScanner.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/TServerUtils.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/VerifyTabletAssignments.java
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java Fri Apr 6 16:35:29 2012
@@ -24,14 +24,13 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.Map.Entry;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -391,7 +390,7 @@ public class TabletLocatorImpl extends T
if (ptl != null) {
List<TabletLocation> locations = locationObtainer.lookupTablet(ptl, metadataRow, lastTabletRow, parent);
- if (locations.size() == 0 && !ptl.tablet_extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ if (locations.size() == 0 && !ptl.tablet_extent.isRootTablet()) {
// try the next tablet
Text er = ptl.tablet_extent.getEndRow();
if (er != null && er.compareTo(lastTabletRow) < 0) {
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletType.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletType.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletType.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/impl/TabletType.java Fri Apr 6 16:35:29 2012
@@ -25,9 +25,9 @@ public enum TabletType {
ROOT, METADATA, USER;
public static TabletType type(KeyExtent ke) {
- if (ke.equals(Constants.ROOT_TABLET_EXTENT))
+ if (ke.isRootTablet())
return ROOT;
- if (ke.getTableId().toString().equals(Constants.METADATA_TABLE_ID))
+ if (ke.isMeta())
return METADATA;
return USER;
}
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java Fri Apr 6 16:35:29 2012
@@ -58,13 +58,8 @@ public enum Property {
"The balancer class that accumulo will use to make tablet assignment and migration decisions."),
MASTER_LOGGER_BALANCER("master.logger.balancer", "org.apache.accumulo.server.master.balancer.SimpleLoggerBalancer", PropertyType.CLASSNAME,
"The balancer class that accumulo will use to make logger assignment decisions."),
- MASTER_RECOVERY_REDUCERS("master.recovery.reducers", "10", PropertyType.COUNT, "Number of reducers to use to sort recovery logs (per log)"),
MASTER_RECOVERY_MAXAGE("master.recovery.max.age", "60m", PropertyType.TIMEDURATION, "Recovery files older than this age will be removed."),
MASTER_RECOVERY_MAXTIME("master.recovery.time.max", "30m", PropertyType.TIMEDURATION, "The maximum time to attempt recovery before giving up"),
- MASTER_RECOVERY_QUEUE("master.recovery.queue", "default", PropertyType.STRING, "Priority queue to use for log recovery map/reduce jobs."),
- MASTER_RECOVERY_POOL("master.recovery.pool", "recovery", PropertyType.STRING, "Priority queue to use for log recovery map/reduce jobs."),
- MASTER_RECOVERY_SORT_MAPREDUCE("master.recovery.sort.mapreduce", "false", PropertyType.BOOLEAN,
- "If true, use map/reduce to sort write-ahead logs during recovery"),
MASTER_BULK_RETRIES("master.bulk.retries", "3", PropertyType.COUNT, "The number of attempts to bulk-load a file before giving up."),
MASTER_BULK_THREADPOOL_SIZE("master.bulk.threadpool.size", "5", PropertyType.COUNT, "The number of threads to use when coordinating a bulk-import."),
MASTER_MINTHREADS("master.server.threads.minimum", "2", PropertyType.COUNT, "The minimum number of threads to use to handle incoming requests."),
Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java Fri Apr 6 16:35:29 2012
@@ -772,6 +772,13 @@ public class KeyExtent implements Writab
return false;
return prevExtent.getEndRow().equals(getPrevEndRow());
-
+ }
+
+ public boolean isMeta() {
+ return getTableId().toString().equals(Constants.METADATA_TABLE_ID);
+ }
+
+ public boolean isRootTablet() {
+ return this.compareTo(Constants.ROOT_TABLET_EXTENT) == 0;
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java Fri Apr 6 16:35:29 2012
@@ -79,7 +79,7 @@ public class Accumulo {
}
}
- public static void init(FileSystem fs, String application) throws UnknownHostException {
+ public static void init(FileSystem fs, ServerConfiguration config, String application) throws UnknownHostException {
System.setProperty("org.apache.accumulo.core.application", application);
@@ -120,7 +120,7 @@ public class Accumulo {
}
TreeMap<String,String> sortedProps = new TreeMap<String,String>();
- for (Entry<String,String> entry : ServerConfiguration.getSystemConfiguration())
+ for (Entry<String,String> entry : config.getConfiguration())
sortedProps.put(entry.getKey(), entry.getValue());
for (Entry<String,String> entry : sortedProps.entrySet()) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java Fri Apr 6 16:35:29 2012
@@ -56,8 +56,10 @@ public class ClientServiceHandler implem
private static final Logger log = Logger.getLogger(ClientServiceHandler.class);
private static Authenticator authenticator = ZKAuthenticator.getInstance();
private final TransactionWatcher transactionWatcher;
+ private final Instance instance;
- public ClientServiceHandler(TransactionWatcher transactionWatcher) {
+ public ClientServiceHandler(Instance instance, TransactionWatcher transactionWatcher) {
+ this.instance = instance;
this.transactionWatcher = transactionWatcher;
}
@@ -85,7 +87,7 @@ public class ClientServiceHandler implem
@Override
public String getZooKeepers() {
- return ServerConfiguration.getSystemConfiguration().get(Property.INSTANCE_ZK_HOST);
+ return instance.getZooKeepers();
}
@Override
@@ -231,7 +233,7 @@ public class ClientServiceHandler implem
public Map<String,String> getConfiguration(ConfigurationType type) throws TException {
switch (type) {
case CURRENT:
- return conf(ServerConfiguration.getSystemConfiguration());
+ return conf(new ServerConfiguration(instance).getConfiguration());
case SITE:
return conf(ServerConfiguration.getSiteConfiguration());
case DEFAULT:
@@ -242,9 +244,8 @@ public class ClientServiceHandler implem
@Override
public Map<String,String> getTableConfiguration(String tableName) throws TException, ThriftTableOperationException {
- Instance instance = HdfsZooInstance.getInstance();
String tableId = checkTableId(tableName, null);
- return conf(ServerConfiguration.getTableConfiguration(instance.getInstanceID(), tableId));
+ return conf(new ServerConfiguration(instance).getTableConfiguration(tableId));
}
@Override
@@ -255,7 +256,7 @@ public class ClientServiceHandler implem
throw new AccumuloSecurityException(credentials.getUser(), SecurityErrorCode.PERMISSION_DENIED);
return transactionWatcher.run(Constants.BULK_ARBITRATOR_TYPE, tid, new Callable<List<String>>() {
public List<String> call() throws Exception {
- return BulkImporter.bulkLoad(ServerConfiguration.getSystemConfiguration(), HdfsZooInstance.getInstance(), credentials, tid, tableId, files, errorDir,
+ return BulkImporter.bulkLoad(new ServerConfiguration(instance).getConfiguration(), instance, credentials, tid, tableId, files, errorDir,
setTime);
}
});
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java Fri Apr 6 16:35:29 2012
@@ -59,7 +59,7 @@ public class HdfsZooInstance implements
}
private HdfsZooInstance() {
- AccumuloConfiguration acuConf = ServerConfiguration.getSystemConfiguration();
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
zooCache = new ZooCache(acuConf.get(Property.INSTANCE_ZK_HOST), (int) acuConf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
}
@@ -131,12 +131,12 @@ public class HdfsZooInstance implements
@Override
public String getZooKeepers() {
- return getConfiguration().get(Property.INSTANCE_ZK_HOST);
+ return ServerConfiguration.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST);
}
@Override
public int getZooKeepersSessionTimeOut() {
- return (int) getConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ return (int) ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
}
@SuppressWarnings("deprecation")
@@ -163,7 +163,7 @@ public class HdfsZooInstance implements
@Override
public AccumuloConfiguration getConfiguration() {
if (conf == null)
- conf = ServerConfiguration.getSystemConfiguration();
+ conf = new ServerConfiguration(this).getConfiguration();
return conf;
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ConfigSanityCheck.java Fri Apr 6 16:35:29 2012
@@ -16,13 +16,15 @@
*/
package org.apache.accumulo.server.conf;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+
public class ConfigSanityCheck {
/**
* @param args
*/
public static void main(String[] args) {
- ServerConfiguration.getSystemConfiguration();
+ new ServerConfiguration(HdfsZooInstance.getInstance()).getConfiguration();
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ServerConfiguration.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ServerConfiguration.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ServerConfiguration.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/ServerConfiguration.java Fri Apr 6 16:35:29 2012
@@ -21,11 +21,12 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigSanityCheck;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
-import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.core.data.KeyExtent;
public class ServerConfiguration {
@@ -44,9 +45,9 @@ public class ServerConfiguration {
}
}
- private static synchronized ZooConfiguration getZooConfiguration() {
+ private static synchronized ZooConfiguration getZooConfiguration(Instance instance) {
checkPermissions();
- return ZooConfiguration.getInstance(getSiteConfiguration());
+ return ZooConfiguration.getInstance(instance, getSiteConfiguration());
}
public static synchronized DefaultConfiguration getDefaultConfiguration() {
@@ -54,16 +55,16 @@ public class ServerConfiguration {
return DefaultConfiguration.getInstance();
}
- public static synchronized AccumuloConfiguration getSystemConfiguration() {
- return getZooConfiguration();
+ public static synchronized AccumuloConfiguration getSystemConfiguration(Instance instance) {
+ return getZooConfiguration(instance);
}
-
- public static TableConfiguration getTableConfiguration(String instanceId, String tableId) {
+
+ public static TableConfiguration getTableConfiguration(Instance instance, String tableId) {
checkPermissions();
synchronized (tableInstances) {
TableConfiguration conf = tableInstances.get(tableId);
if (conf == null) {
- conf = new TableConfiguration(instanceId, tableId, getSystemConfiguration());
+ conf = new TableConfiguration(instance.getInstanceID(), tableId, getSystemConfiguration(instance));
ConfigSanityCheck.validate(conf);
tableInstances.put(tableId, conf);
}
@@ -71,10 +72,6 @@ public class ServerConfiguration {
}
}
- public static TableConfiguration getTableConfiguration(String tableId) {
- return getTableConfiguration(HdfsZooInstance.getInstance().getInstanceID(), tableId);
- }
-
static void removeTableIdInstance(String tableId) {
synchronized (tableInstances) {
tableInstances.remove(tableId);
@@ -88,4 +85,27 @@ public class ServerConfiguration {
}
}
}
+
+ private final Instance instance;
+
+ public ServerConfiguration(Instance instance) {
+ this.instance = instance;
+ }
+
+ public TableConfiguration getTableConfiguration(String tableId) {
+ return getTableConfiguration(instance, tableId);
+ }
+
+ public TableConfiguration getTableConfiguration(KeyExtent extent) {
+ return getTableConfiguration(extent.getTableId().toString());
+ }
+
+ public synchronized AccumuloConfiguration getConfiguration() {
+ return getZooConfiguration(instance);
+ }
+
+ public Instance getInstance() {
+ return instance;
+ }
+
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfWatcher.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfWatcher.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfWatcher.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfWatcher.java Fri Apr 6 16:35:29 2012
@@ -17,6 +17,7 @@
package org.apache.accumulo.server.conf;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -30,10 +31,10 @@ class TableConfWatcher implements Watche
}
private static final Logger log = Logger.getLogger(TableConfWatcher.class);
- private String instanceId = null;
+ private Instance instance = null;
- TableConfWatcher(String instanceId) {
- this.instanceId = instanceId;
+ TableConfWatcher(Instance instance) {
+ this.instance = instance;
}
@Override
@@ -42,7 +43,7 @@ class TableConfWatcher implements Watche
if (log.isTraceEnabled())
log.trace("WatchEvent : " + path + " " + event.getState() + " " + event.getType());
- String tablesPrefix = ZooUtil.getRoot(instanceId) + Constants.ZTABLES + "/";
+ String tablesPrefix = ZooUtil.getRoot(instance) + Constants.ZTABLES + "/";
String tableId = null;
String key = null;
@@ -68,10 +69,10 @@ class TableConfWatcher implements Watche
if (log.isTraceEnabled())
log.trace("EventNodeDataChanged " + event.getPath());
if (key != null)
- ServerConfiguration.getTableConfiguration(instanceId, tableId).propertyChanged(key);
+ ServerConfiguration.getTableConfiguration(instance, tableId).propertyChanged(key);
break;
case NodeChildrenChanged:
- ServerConfiguration.getTableConfiguration(instanceId, tableId).propertiesChanged(key);
+ ServerConfiguration.getTableConfiguration(instance, tableId).propertiesChanged(key);
break;
case NodeDeleted:
if (key == null) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/conf/TableConfiguration.java Fri Apr 6 16:35:29 2012
@@ -21,9 +21,9 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
-import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
@@ -61,7 +61,7 @@ public class TableConfiguration extends
if (tablePropCache == null)
synchronized (TableConfiguration.class) {
if (tablePropCache == null)
- tablePropCache = new ZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), new TableConfWatcher(inst.getInstanceID()));
+ tablePropCache = new ZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), new TableConfWatcher(inst));
}
return tablePropCache;
}
@@ -147,4 +147,8 @@ public class TableConfiguration extends
return entries.entrySet().iterator();
}
+
+ public String getTableId() {
+ return table;
+ }
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java Fri Apr 6 16:35:29 2012
@@ -19,28 +19,24 @@ package org.apache.accumulo.server.const
import java.io.IOException;
import java.util.Map.Entry;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.constraints.Constraint;
-import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
import org.apache.log4j.Logger;
public class ConstraintLoader {
private static final Logger log = Logger.getLogger(ConstraintLoader.class);
- public static ConstraintChecker load(String table) throws IOException {
+ public static ConstraintChecker load(TableConfiguration conf) throws IOException {
try {
-
- AccumuloConfiguration conf = ServerConfiguration.getTableConfiguration(table);
-
ConstraintChecker cc = new ConstraintChecker();
for (Entry<String,String> entry : conf) {
if (entry.getKey().startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey())) {
String className = entry.getValue();
Class<? extends Constraint> clazz = AccumuloClassLoader.loadClass(className, Constraint.class);
- log.debug("Loaded constraint " + clazz.getName() + " for " + table);
+ log.debug("Loaded constraint " + clazz.getName() + " for " + conf.getTableId());
cc.addConstraint(clazz.newInstance());
}
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java Fri Apr 6 16:35:29 2012
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.accumulo.cloudtrace.instrument.Span;
import org.apache.accumulo.cloudtrace.instrument.Trace;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
@@ -40,7 +41,6 @@ import org.apache.accumulo.core.util.Thr
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.accumulo.server.util.MetadataTable;
import org.apache.accumulo.server.util.MetadataTable.LogEntry;
@@ -58,7 +58,15 @@ import org.apache.zookeeper.KeeperExcept
public class GarbageCollectWriteAheadLogs {
private static final Logger log = Logger.getLogger(GarbageCollectWriteAheadLogs.class);
- public static void collect(FileSystem fs, GCStatus status) {
+ private final AccumuloConfiguration conf;
+ private final FileSystem fs;
+
+ GarbageCollectWriteAheadLogs(FileSystem fs, AccumuloConfiguration conf) {
+ this.fs = fs;
+ this.conf = conf;
+ }
+
+ public void collect(GCStatus status) {
Span span = Trace.start("scanServers");
try {
@@ -88,7 +96,7 @@ public class GarbageCollectWriteAheadLog
span = Trace.start("removeFiles");
Map<String,ArrayList<String>> serverToFileMap = mapServersToFiles(fileToServerMap);
- count = removeFiles(fs, serverToFileMap, status);
+ count = removeFiles(serverToFileMap, status);
long removeStop = System.currentTimeMillis();
log.info(String.format("%d total logs removed from %d servers in %.2f seconds", count, serverToFileMap.size(), (removeStop - logEntryScanStop) / 1000.));
@@ -103,7 +111,7 @@ public class GarbageCollectWriteAheadLog
}
}
- private static int removeFiles(final FileSystem fs, Map<String,ArrayList<String>> serverToFileMap, final GCStatus status) {
+ private int removeFiles(Map<String,ArrayList<String>> serverToFileMap, final GCStatus status) {
final AtomicInteger count = new AtomicInteger();
ExecutorService threadPool = java.util.concurrent.Executors.newCachedThreadPool();
@@ -114,8 +122,7 @@ public class GarbageCollectWriteAheadLog
@Override
public void run() {
try {
- Iface logger = ThriftUtil.getClient(new MutationLogger.Client.Factory(), server, Property.LOGGER_PORT, Property.TSERV_LOGGER_TIMEOUT,
- ServerConfiguration.getSystemConfiguration());
+ Iface logger = ThriftUtil.getClient(new MutationLogger.Client.Factory(), server, Property.LOGGER_PORT, Property.TSERV_LOGGER_TIMEOUT, conf);
try {
count.addAndGet(files.size());
log.debug(String.format("removing %d files from %s", files.size(), server));
@@ -186,7 +193,7 @@ public class GarbageCollectWriteAheadLog
return count;
}
- private static int scanServers(Map<String,String> fileToServerMap) throws Exception {
+ private int scanServers(Map<String,String> fileToServerMap) throws Exception {
int count = 0;
IZooReaderWriter zk = ZooReaderWriter.getInstance();
String loggersDir = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZLOGGERS;
@@ -198,8 +205,7 @@ public class GarbageCollectWriteAheadLog
try {
byte[] data = zk.getData(loggersDir + "/" + server, null);
address = new String(data);
- Iface logger = ThriftUtil.getClient(new MutationLogger.Client.Factory(), address, Property.LOGGER_PORT, Property.TSERV_LOGGER_TIMEOUT,
- ServerConfiguration.getSystemConfiguration());
+ Iface logger = ThriftUtil.getClient(new MutationLogger.Client.Factory(), address, Property.LOGGER_PORT, Property.TSERV_LOGGER_TIMEOUT, conf);
for (String log : logger.getClosedLogs(null, SecurityConstants.getSystemCredentials())) {
fileToServerMap.put(log, address);
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Fri Apr 6 16:35:29 2012
@@ -48,7 +48,6 @@ import org.apache.accumulo.core.client.M
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.Tables;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
@@ -118,7 +117,6 @@ public class SimpleGarbageCollector impl
private static final Logger log = Logger.getLogger(SimpleGarbageCollector.class);
- private Instance instance;
private AuthInfo credentials;
private long gcStartDelay;
private boolean checkForBulkProcessingFiles;
@@ -132,9 +130,13 @@ public class SimpleGarbageCollector impl
private int numDeleteThreads;
+ private ServerConfiguration conf;
+
public static void main(String[] args) throws UnknownHostException, IOException {
- final FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration());
- Accumulo.init(fs, "gc");
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration serverConf = new ServerConfiguration(instance);
+ final FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), serverConf.getConfiguration());
+ Accumulo.init(fs, serverConf, "gc");
String address = "localhost";
SimpleGarbageCollector gc = new SimpleGarbageCollector();
try {
@@ -157,7 +159,7 @@ public class SimpleGarbageCollector impl
throw new IllegalArgumentException(str, e);
}
- gc.init(fs, HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), ServerConfiguration.getSystemConfiguration());
+ gc.init(fs, serverConf, SecurityConstants.getSystemCredentials());
Accumulo.enableTracing(address, "gc");
gc.run();
}
@@ -181,14 +183,14 @@ public class SimpleGarbageCollector impl
this.address = address;
}
- public void init(FileSystem fs, Instance instance, AuthInfo credentials, AccumuloConfiguration conf) {
+ public void init(FileSystem fs, ServerConfiguration conf, AuthInfo credentials) {
this.fs = TraceFileSystem.wrap(fs);
- this.instance = instance;
this.credentials = credentials;
+ this.conf = conf;
- gcStartDelay = conf.getTimeInMillis(Property.GC_CYCLE_START);
- long gcDelay = conf.getTimeInMillis(Property.GC_CYCLE_DELAY);
- numDeleteThreads = conf.getCount(Property.GC_DELETE_THREADS);
+ gcStartDelay = conf.getConfiguration().getTimeInMillis(Property.GC_CYCLE_START);
+ long gcDelay = conf.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
+ numDeleteThreads = conf.getConfiguration().getCount(Property.GC_DELETE_THREADS);
log.info("start delay: " + (offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds"));
log.info("time delay: " + gcDelay + " milliseconds");
log.info("safemode: " + safemode);
@@ -294,9 +296,10 @@ public class SimpleGarbageCollector impl
// Clean up any unused write-ahead logs
Span waLogs = Trace.start("walogs");
+ GarbageCollectWriteAheadLogs walogCollector = new GarbageCollectWriteAheadLogs(fs, conf.getConfiguration());
try {
log.info("Beginning garbage collection of write-ahead logs");
- GarbageCollectWriteAheadLogs.collect(fs, status);
+ walogCollector.collect(status);
} catch (Exception e) {
log.error(e, e);
}
@@ -305,7 +308,7 @@ public class SimpleGarbageCollector impl
Trace.offNoFlush();
try {
- long gcDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
+ long gcDelay = conf.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
log.debug("Sleeping for " + gcDelay + " milliseconds");
Thread.sleep(gcDelay);
} catch (InterruptedException e) {
@@ -330,8 +333,8 @@ public class SimpleGarbageCollector impl
}
}
- Tables.clearCache(instance);
- Set<String> tableIdsInZookeeper = Tables.getIdToNameMap(instance).keySet();
+ Tables.clearCache(conf.getInstance());
+ Set<String> tableIdsInZookeeper = Tables.getIdToNameMap(conf.getInstance()).keySet();
tableIdsWithDeletes.removeAll(tableIdsInZookeeper);
@@ -371,7 +374,7 @@ public class SimpleGarbageCollector impl
private InetSocketAddress startStatsService() throws UnknownHostException {
GCMonitorService.Processor processor = new GCMonitorService.Processor(TraceWrap.service(this));
- int port = ServerConfiguration.getSystemConfiguration().getPort(Property.GC_PORT);
+ int port = conf.getConfiguration().getPort(Property.GC_PORT);
try {
TServerUtils.startTServer(port, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000);
} catch (Exception ex) {
@@ -406,7 +409,7 @@ public class SimpleGarbageCollector impl
return candidates;
}
- Scanner scanner = instance.getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ Scanner scanner = conf.getInstance().getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
if (continueKey != null) {
// want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them, then will never inspect deletes after N
@@ -448,13 +451,13 @@ public class SimpleGarbageCollector impl
Scanner scanner;
if (offline) {
try {
- scanner = new OfflineMetadataScanner();
+ scanner = new OfflineMetadataScanner(conf.getConfiguration(), fs);
} catch (IOException e) {
throw new IllegalStateException("Unable to create offline metadata scanner", e);
}
} else {
try {
- scanner = new IsolatedScanner(instance.getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
+ scanner = new IsolatedScanner(conf.getInstance().getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
} catch (AccumuloSecurityException ex) {
throw new AccumuloException(ex);
} catch (TableNotFoundException ex) {
@@ -545,7 +548,7 @@ public class SimpleGarbageCollector impl
if (!offline) {
Connector c;
try {
- c = instance.getConnector(SecurityConstants.getSystemCredentials());
+ c = conf.getInstance().getConnector(SecurityConstants.getSystemCredentials());
writer = c.createBatchWriter(Constants.METADATA_TABLE_NAME, 10000000, 60000l, 3);
} catch (Exception e) {
log.error("Unable to create writer to remove file from the !METADATA table", e);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/logger/LogService.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/logger/LogService.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/logger/LogService.java Fri Apr 6 16:35:29 2012
@@ -114,12 +114,14 @@ public class LogService implements Mutat
public static void main(String[] args) throws Exception {
LogService logService;
- FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration());
- Accumulo.init(fs, "logger");
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), conf.getConfiguration());
+ Accumulo.init(fs, conf, "logger");
String hostname = Accumulo.getLocalAddress(args);
try {
- logService = new LogService(HdfsZooInstance.getInstance(), fs, hostname);
+ logService = new LogService(conf, fs, hostname);
} catch (Exception e) {
LOG.fatal("Failed to initialize log service args=" + Arrays.asList(args), e);
throw e;
@@ -132,13 +134,14 @@ public class LogService implements Mutat
}
}
- public LogService(Instance instance, FileSystem fs, String hostname) throws UnknownHostException, KeeperException, InterruptedException, IOException {
- this.instance = instance;
- FileSystemMonitor.start(Property.LOGGER_MONITOR_FS);
+ public LogService(ServerConfiguration config, FileSystem fs, String hostname) throws UnknownHostException, KeeperException, InterruptedException, IOException {
+ this.instance = config.getInstance();
+ AccumuloConfiguration acuConf = config.getConfiguration();
+ FileSystemMonitor.start(acuConf, Property.LOGGER_MONITOR_FS);
fs = TraceFileSystem.wrap(fs);
final Set<String> rootDirs = new HashSet<String>();
- for (String root : ServerConfiguration.getSystemConfiguration().get(Property.LOGGER_DIR).split(",")) {
+ for (String root : acuConf.get(Property.LOGGER_DIR).split(",")) {
if (!root.startsWith("/"))
root = System.getenv("ACCUMULO_HOME") + "/" + root;
else if (root.equals(""))
@@ -173,9 +176,8 @@ public class LogService implements Mutat
}
authenticator = ZKAuthenticator.getInstance();
- int poolSize = ServerConfiguration.getSystemConfiguration().getCount(Property.LOGGER_COPY_THREADPOOL_SIZE);
- boolean archive = ServerConfiguration.getSystemConfiguration().getBoolean(Property.LOGGER_ARCHIVE);
- AccumuloConfiguration acuConf = ServerConfiguration.getSystemConfiguration();
+ int poolSize = acuConf.getCount(Property.LOGGER_COPY_THREADPOOL_SIZE);
+ boolean archive = acuConf.getBoolean(Property.LOGGER_ARCHIVE);
writer_ = new LogWriter(acuConf, fs, rootDirs, instance.getInstanceID(), poolSize, archive);
InvocationHandler h = new InvocationHandler() {
@Override
@@ -210,7 +212,7 @@ public class LogService implements Mutat
writer = (MutationLogger.Iface) Proxy.newProxyInstance(MutationLogger.Iface.class.getClassLoader(), new Class[] {MutationLogger.Iface.class}, h);
// Create the thrift-based logging service
MutationLogger.Processor processor = new MutationLogger.Processor(TraceWrap.service(this));
- ServerPort sp = TServerUtils.startServer(Property.LOGGER_PORT, processor, this.getClass().getSimpleName(), "Logger Client Service Handler",
+ ServerPort sp = TServerUtils.startServer(acuConf, Property.LOGGER_PORT, processor, this.getClass().getSimpleName(), "Logger Client Service Handler",
Property.LOGGER_PORTSEARCH, Property.LOGGER_MINTHREADS, Property.LOGGER_THREADCHECK);
service = sp.server;
InetSocketAddress address = new InetSocketAddress(hostname, sp.port);
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java Fri Apr 6 16:35:29 2012
@@ -17,8 +17,6 @@
package org.apache.accumulo.server.master;
import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -26,28 +24,18 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.master.thrift.RecoveryStatus;
import org.apache.accumulo.core.security.thrift.AuthInfo;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.server.ServerConstants;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.tabletserver.log.RemoteLogger;
-import org.apache.accumulo.server.trace.TraceFileSystem;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.log4j.Logger;
public class CoordinateRecoveryTask implements Runnable {
@@ -115,24 +103,24 @@ public class CoordinateRecoveryTask impl
final LogFile logFile;
final long copyStartTime;
long copySize = 0;
- Job sortJob = null;
- boolean useMapReduce = ServerConfiguration.getSystemConfiguration().getBoolean(Property.MASTER_RECOVERY_SORT_MAPREDUCE);
JobComplete notify = null;
+ final AccumuloConfiguration config;
- RecoveryJob(LogFile entry, JobComplete callback) throws Exception {
+ RecoveryJob(LogFile entry, JobComplete callback, AccumuloConfiguration conf) throws Exception {
logFile = entry;
copyStartTime = System.currentTimeMillis();
notify = callback;
+ config = conf;
}
private void startCopy() throws Exception {
log.debug("Starting log recovery: " + logFile);
try {
// Ask the logging server to put the file in HDFS
- RemoteLogger logger = new RemoteLogger(logFile.server);
+ RemoteLogger logger = new RemoteLogger(logFile.server, config);
String base = logFile.unsortedFileName();
log.debug("Starting to copy " + logFile.file + " from " + logFile.server);
- copySize = logger.startCopy(logFile.file, base, !useMapReduce);
+ copySize = logger.startCopy(logFile.file, base);
} catch (Throwable t) {
log.warn("Unable to recover " + logFile + "(" + t + ")", t);
fail();
@@ -140,20 +128,6 @@ public class CoordinateRecoveryTask impl
}
- synchronized private void startSort() throws Exception {
- Integer reducers = ServerConfiguration.getSystemConfiguration().getCount(Property.MASTER_RECOVERY_REDUCERS);
- String queue = ServerConfiguration.getSystemConfiguration().get(Property.MASTER_RECOVERY_QUEUE);
- String pool = ServerConfiguration.getSystemConfiguration().get(Property.MASTER_RECOVERY_POOL);
- String result = logFile.recoveryFileName();
- fs.delete(new Path(result), true);
- List<String> jars = new ArrayList<String>();
- jars.addAll(jarsLike("accumulo-core"));
- jars.addAll(jarsLike("zookeeper"));
- jars.addAll(jarsLike("libthrift"));
- sortJob = LogSort.startSort(true,
- new String[] {"-libjars", StringUtil.join(jars, ","), "-r", reducers.toString(), "-q", queue, "-p", pool, logFile.unsortedFileName(), result});
- }
-
synchronized boolean isComplete() throws Exception {
if (fs.exists(new Path(logFile.successFileName()))) {
return true;
@@ -162,28 +136,17 @@ public class CoordinateRecoveryTask impl
return true;
}
- if (elapsedMillis() > ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.MASTER_RECOVERY_MAXTIME)) {
+ if (elapsedMillis() > config.getTimeInMillis(Property.MASTER_RECOVERY_MAXTIME)) {
log.warn("Recovery taking too long, giving up");
- if (sortJob != null)
- sortJob.killJob();
return true;
}
// Did the sort fail?
- if (sortJob == null && fs.exists(new Path(logFile.failedFileName()))) {
+ if (fs.exists(new Path(logFile.failedFileName()))) {
return true;
}
log.debug(toString());
-
- // Did the copy complete?
- if (useMapReduce && fs.exists(new Path(logFile.unsortedFileName()))) {
- // Start the sort or check on it
- if (sortJob == null) {
- log.debug("Finished copy of " + logFile.file + " from " + logFile.server + ": took " + (elapsedMillis() / 1000.) + " seconds, starting sort");
- startSort();
- }
- }
return false;
}
@@ -192,7 +155,6 @@ public class CoordinateRecoveryTask impl
}
synchronized void fail(boolean createFailFlag) {
- sortJob = null;
String failed = logFile.failedFileName();
try {
if (createFailFlag)
@@ -208,29 +170,14 @@ public class CoordinateRecoveryTask impl
}
synchronized public String toString() {
- if (sortJob != null) {
- try {
- return String.format("Sorting log %s job %s: %2.1f/%2.1f", logFile.file, sortJob.getTrackingURL(), sortJob.mapProgress() * 100,
- sortJob.reduceProgress() * 100);
- } catch (Exception e) {
- log.debug("Unable to get stats for sort of " + logFile.file, e);
- }
- }
return String.format("Copying %s from %s (for %f seconds) %2.1f", logFile.file, logFile.server, elapsedMillis() / 1000., copiedSoFar() * 100. / copySize);
}
synchronized long copiedSoFar() {
try {
- if (useMapReduce) {
- Path unsorted = new Path(logFile.unsortedFileName());
- if (fs.exists(unsorted))
- return fs.getFileStatus(unsorted).getLen();
- return fs.getFileStatus(new Path(logFile.copyTempFileName())).getLen();
- } else {
- ContentSummary contentSummary = fs.getContentSummary(new Path(logFile.recoveryFileName()));
- // map files are bigger than sequence files
- return (long) (contentSummary.getSpaceConsumed() * .8);
- }
+ ContentSummary contentSummary = fs.getContentSummary(new Path(logFile.recoveryFileName()));
+ // map files are bigger than sequence files
+ return (long) (contentSummary.getSpaceConsumed() * .8);
} catch (Exception ex) {
return 0;
}
@@ -238,17 +185,17 @@ public class CoordinateRecoveryTask impl
synchronized public RecoveryStatus getStatus() throws IOException {
try {
- return new RecoveryStatus(logFile.server, logFile.file, (sortJob == null ? 0. : sortJob.mapProgress()), (sortJob == null ? 0.
- : sortJob.reduceProgress()), (int) (System.currentTimeMillis() - copyStartTime), (sortJob != null) ? 1. : (copySize == 0 ? 0 : copiedSoFar()
- / (double) copySize));
+ return new RecoveryStatus(logFile.server, logFile.file, 0., 0., (int) (System.currentTimeMillis() - copyStartTime), (copiedSoFar() / (double) copySize));
} catch (Exception e) {
return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
}
}
}
- public CoordinateRecoveryTask(FileSystem fs) {
+ AccumuloConfiguration config;
+ public CoordinateRecoveryTask(FileSystem fs, AccumuloConfiguration conf) {
this.fs = fs;
+ this.config = conf;
}
public boolean recover(AuthInfo credentials, KeyExtent extent, Collection<Collection<String>> entries, JobComplete notify) {
@@ -282,7 +229,7 @@ public class CoordinateRecoveryTask impl
try {
synchronized (processing) {
if (!fs.exists(new Path(failed)) && !fs.exists(new Path(recovered)) && !processing.containsKey(metadataEntry)) {
- processing.put(metadataEntry, job = new RecoveryJob(logFile, notify));
+ processing.put(metadataEntry, job = new RecoveryJob(logFile, notify, config));
}
}
if (job != null) {
@@ -296,58 +243,10 @@ public class CoordinateRecoveryTask impl
return finished;
}
- private static List<String> jarsLike(String substr) {
- ArrayList<String> result = new ArrayList<String>();
- URLClassLoader loader = (URLClassLoader) CoordinateRecoveryTask.class.getClassLoader();
- for (URL url : loader.getURLs()) {
- String path = url.getPath();
- if (path.indexOf(substr) >= 0 && path.endsWith(".jar") && path.indexOf("javadoc") < 0 && path.indexOf("sources") < 0) {
- result.add(path);
- }
- }
- return result;
- }
-
- void cleanupOldJobs() {
- try {
- Configuration conf = CachedConfiguration.getInstance();
- @SuppressWarnings("deprecation")
- // No alternative api in hadoop 20
- JobClient jc = new JobClient(new org.apache.hadoop.mapred.JobConf(conf));
- for (JobStatus status : jc.getAllJobs()) {
- if (!status.isJobComplete()) {
- RunningJob job = jc.getJob(status.getJobID());
- if (job.getJobName().equals(LogSort.getJobName())) {
- log.info("found a running " + job.getJobName());
- Configuration jobConfig = new Configuration(false);
- log.info("fetching configuration from " + job.getJobFile());
- jobConfig.addResource(TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration())).open(
- new Path(job.getJobFile())));
- if (HdfsZooInstance.getInstance().getInstanceID().equals(jobConfig.get(LogSort.INSTANCE_ID_PROPERTY))) {
- log.info("Killing job " + job.getID().toString());
- }
- }
- }
- }
- FileStatus[] children = fs.listStatus(new Path(ServerConstants.getRecoveryDir()));
- if (children != null) {
- for (FileStatus child : children) {
- log.info("Deleting recovery directory " + child);
- fs.delete(child.getPath(), true);
- }
- }
- } catch (IOException e) {
- log.error("Error cleaning up old Log Sort jobs" + e);
- } catch (Exception e) {
- log.error("Unknown error cleaning up old jobs", e);
- }
- }
-
@Override
public void run() {
// Check on the asynchronous requests: keep them moving along
int count = 0;
- cleanupOldJobs();
while (!stop) {
try {
synchronized (processing) {
@@ -381,7 +280,7 @@ public class CoordinateRecoveryTask impl
private void removeOldRecoverFiles() throws IOException {
long now = System.currentTimeMillis();
- long maxAgeInMillis = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.MASTER_RECOVERY_MAXAGE);
+ long maxAgeInMillis = config.getTimeInMillis(Property.MASTER_RECOVERY_MAXAGE);
FileStatus[] children = fs.listStatus(new Path(ServerConstants.getRecoveryDir()));
if (children != null) {
for (FileStatus child : children) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/LiveTServerSet.java Fri Apr 6 16:35:29 2012
@@ -28,6 +28,7 @@ import java.util.TimerTask;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
@@ -37,7 +38,6 @@ import org.apache.accumulo.core.tabletse
import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.security.SecurityConstants;
import org.apache.accumulo.server.util.AddressUtil;
@@ -64,6 +64,7 @@ public class LiveTServerSet implements W
private final Listener cback;
private final Instance instance;
+ private final AccumuloConfiguration conf;
private ZooCache zooCache;
public class TServerConnection {
@@ -78,7 +79,7 @@ public class LiveTServerSet implements W
}
public void assignTablet(ZooLock lock, KeyExtent extent) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.loadTablet(null, SecurityConstants.getSystemCredentials(), lockString(lock), extent.toThrift());
} finally {
@@ -87,7 +88,7 @@ public class LiveTServerSet implements W
}
public void unloadTablet(ZooLock lock, KeyExtent extent, boolean save) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.unloadTablet(null, SecurityConstants.getSystemCredentials(), lockString(lock), extent.toThrift(), save);
} finally {
@@ -96,7 +97,7 @@ public class LiveTServerSet implements W
}
public TabletServerStatus getTableMap() throws TException, ThriftSecurityException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
return client.getTabletServerStatus(null, SecurityConstants.getSystemCredentials());
} finally {
@@ -105,7 +106,7 @@ public class LiveTServerSet implements W
}
public void halt(ZooLock lock) throws TException, ThriftSecurityException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.halt(null, SecurityConstants.getSystemCredentials(), lockString(lock));
} finally {
@@ -114,7 +115,7 @@ public class LiveTServerSet implements W
}
public void fastHalt(ZooLock lock) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.fastHalt(null, SecurityConstants.getSystemCredentials(), lockString(lock));
} finally {
@@ -123,7 +124,7 @@ public class LiveTServerSet implements W
}
public void flush(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.flush(null, SecurityConstants.getSystemCredentials(), lockString(lock), tableId, startRow == null ? null : ByteBuffer.wrap(startRow),
endRow == null ? null : ByteBuffer.wrap(endRow));
@@ -133,7 +134,7 @@ public class LiveTServerSet implements W
}
public void useLoggers(Set<String> loggers) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.useLoggers(null, SecurityConstants.getSystemCredentials(), loggers);
} finally {
@@ -142,7 +143,7 @@ public class LiveTServerSet implements W
}
public void chop(ZooLock lock, KeyExtent extent) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.chop(null, SecurityConstants.getSystemCredentials(), lockString(lock), extent.toThrift());
} finally {
@@ -151,7 +152,7 @@ public class LiveTServerSet implements W
}
public void splitTablet(ZooLock lock, KeyExtent extent, Text splitPoint) throws TException, ThriftSecurityException, NotServingTabletException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client
.splitTablet(null, SecurityConstants.getSystemCredentials(), extent.toThrift(), ByteBuffer.wrap(splitPoint.getBytes(), 0, splitPoint.getLength()));
@@ -161,7 +162,7 @@ public class LiveTServerSet implements W
}
public void flushTablet(ZooLock lock, KeyExtent extent) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.flushTablet(null, SecurityConstants.getSystemCredentials(), lockString(lock), extent.toThrift());
} finally {
@@ -170,7 +171,7 @@ public class LiveTServerSet implements W
}
public void compact(ZooLock lock, String tableId, byte[] startRow, byte[] endRow) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
client.compact(null, SecurityConstants.getSystemCredentials(), lockString(lock), tableId, startRow == null ? null : ByteBuffer.wrap(startRow),
endRow == null ? null : ByteBuffer.wrap(endRow));
@@ -180,7 +181,7 @@ public class LiveTServerSet implements W
}
public boolean isActive(long tid) throws TException {
- TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, ServerConfiguration.getSystemConfiguration());
+ TabletClientService.Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
try {
return client.isActive(null, tid);
} finally {
@@ -210,10 +211,11 @@ public class LiveTServerSet implements W
// Map from tserver master service to server information
private Map<String,TServerInfo> current = new HashMap<String,TServerInfo>();
-
- public LiveTServerSet(Instance instance, Listener cback) {
+
+ public LiveTServerSet(Instance instance, AccumuloConfiguration conf, Listener cback) {
this.cback = cback;
this.instance = instance;
+ this.conf = conf;
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Fri Apr 6 16:35:29 2012
@@ -56,6 +56,7 @@ import org.apache.accumulo.core.client.i
import org.apache.accumulo.core.client.impl.thrift.TableOperation;
import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType;
import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
@@ -331,6 +332,8 @@ public class Master implements LiveTServ
private AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
+ private ServerConfiguration serverConfig;
+
private void upgradeMetadata() {
if (Accumulo.getAccumuloPersistentVersion(fs) == Constants.PREV_DATA_VERSION) {
if (upgradeMetadataRunning.compareAndSet(false, true)) {
@@ -344,7 +347,7 @@ public class Master implements LiveTServ
BatchWriter bw = getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME, 10000000, 60000l, 4);
- FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(ServerConfiguration.getSystemConfiguration()) + "/*"));
+ FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(getSystemConfiguration()) + "/*"));
for (FileStatus tableDir : tables) {
FileStatus[] bulkDirs = fs.globStatus(new Path(tableDir.getPath() + "/bulk_*"));
for (FileStatus bulkDir : bulkDirs) {
@@ -509,8 +512,8 @@ public class Master implements LiveTServ
}
// @TODO: maybe move this to Property? We do this in TabletServer, Master, TableLoadBalancer, etc.
- public static <T> T createInstanceFromPropertyName(Property property, Class<T> base, T defaultInstance) {
- String clazzName = ServerConfiguration.getSystemConfiguration().get(property);
+ public static <T> T createInstanceFromPropertyName(AccumuloConfiguration conf, Property property, Class<T> base, T defaultInstance) {
+ String clazzName = conf.get(property);
T instance = null;
try {
@@ -528,18 +531,22 @@ public class Master implements LiveTServ
return instance;
}
- public Master(Instance instance, FileSystem fs, String hostname) throws IOException {
- this.instance = instance;
+ public Master(ServerConfiguration config, FileSystem fs, String hostname) throws IOException {
+ this.serverConfig = config;
+ this.instance = config.getInstance();
this.fs = TraceFileSystem.wrap(fs);
this.hostname = hostname;
+
+ AccumuloConfiguration aconf = serverConfig.getConfiguration();
log.info("Version " + Constants.VERSION);
log.info("Instance " + instance.getInstanceID());
- ThriftTransportPool.getInstance().setIdleTime(ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+ ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
authenticator = ZKAuthenticator.getInstance();
- tserverSet = new LiveTServerSet(instance, this);
- this.tabletBalancer = createInstanceFromPropertyName(Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
- this.loggerBalancer = createInstanceFromPropertyName(Property.MASTER_LOGGER_BALANCER, LoggerBalancer.class, new SimpleLoggerBalancer());
+ tserverSet = new LiveTServerSet(instance, config.getConfiguration(), this);
+ this.tabletBalancer = createInstanceFromPropertyName(aconf, Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
+ this.tabletBalancer.init(serverConfig);
+ this.loggerBalancer = createInstanceFromPropertyName(aconf, Property.MASTER_LOGGER_BALANCER, LoggerBalancer.class, new SimpleLoggerBalancer());
}
public TServerConnection getConnection(TServerInstance server) {
@@ -1173,11 +1180,11 @@ public class Master implements LiveTServ
case HAVE_LOCK: // fall-through intended
case INITIAL: // fall-through intended
case SAFE_MODE:
- if (tls.extent.getTableId().equals(METADATA_TABLE_ID))
+ if (tls.extent.isMeta())
return TabletGoalState.HOSTED;
return TabletGoalState.UNASSIGNED;
case UNLOAD_METADATA_TABLETS:
- if (tls.extent.equals(Constants.ROOT_TABLET_EXTENT))
+ if (tls.extent.isRootTablet())
return TabletGoalState.HOSTED;
return TabletGoalState.UNASSIGNED;
case UNLOAD_ROOT_TABLET:
@@ -1929,7 +1936,7 @@ public class Master implements LiveTServ
TServerInstance instance = null;
int crazyHoldTime = 0;
int someHoldTime = 0;
- final long maxWait = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.TSERV_HOLD_TIME_SUICIDE);
+ final long maxWait = getSystemConfiguration().getTimeInMillis(Property.TSERV_HOLD_TIME_SUICIDE);
for (Entry<TServerInstance,TabletServerStatus> entry : tserverStatus.entrySet()) {
if (entry.getValue().getHoldTime() > 0) {
someHoldTime++;
@@ -1959,7 +1966,7 @@ public class Master implements LiveTServ
}
List<String> logNames = new ArrayList<String>(loggers.getLoggersFromZooKeeper().values());
Map<LoggerUser,List<String>> assignmentsOut = new HashMap<LoggerUser,List<String>>();
- int loggersPerServer = ServerConfiguration.getSystemConfiguration().getCount(Property.TSERV_LOGGER_COUNT);
+ int loggersPerServer = getSystemConfiguration().getCount(Property.TSERV_LOGGER_COUNT);
loggerBalancer.balance(logUsers, logNames, assignmentsOut, loggersPerServer);
for (Entry<LoggerUser,List<String>> entry : assignmentsOut.entrySet()) {
TServerUsesLoggers tserver = (TServerUsesLoggers) entry.getKey();
@@ -2040,11 +2047,11 @@ public class Master implements LiveTServ
TableManager.getInstance().addObserver(this);
- recovery = new CoordinateRecoveryTask(fs);
+ recovery = new CoordinateRecoveryTask(fs, getSystemConfiguration());
Thread recoveryThread = new Daemon(new LoggingRunnable(log, recovery), "Recovery Status");
recoveryThread.start();
- loggers = new TabletServerLoggers(this, ServerConfiguration.getSystemConfiguration());
+ loggers = new TabletServerLoggers(this, getSystemConfiguration());
loggers.scanZooKeeperForUpdates();
StatusThread statusThread = new StatusThread();
@@ -2076,7 +2083,7 @@ public class Master implements LiveTServ
}
Processor processor = new MasterClientService.Processor(TraceWrap.service(new MasterClientServiceHandler()));
- clientService = TServerUtils.startServer(Property.MASTER_CLIENTPORT, processor, "Master", "Master Client Service Handler", null,
+ clientService = TServerUtils.startServer(getSystemConfiguration(), Property.MASTER_CLIENTPORT, processor, "Master", "Master Client Service Handler", null,
Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK).server;
while (!clientService.isServing()) {
@@ -2116,8 +2123,8 @@ public class Master implements LiveTServ
}
};
long current = System.currentTimeMillis();
- final long waitTime = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
- final String masterClientAddress = hostname + ":" + ServerConfiguration.getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
+ final long waitTime = getSystemConfiguration().getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
+ final String masterClientAddress = hostname + ":" + getSystemConfiguration().getPort(Property.MASTER_CLIENTPORT);
boolean locked = false;
while (System.currentTimeMillis() - current < waitTime) {
@@ -2144,9 +2151,11 @@ public class Master implements LiveTServ
public static void main(String[] args) throws Exception {
try {
FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration());
- Accumulo.init(fs, "master");
String hostname = Accumulo.getLocalAddress(args);
- Master master = new Master(HdfsZooInstance.getInstance(), fs, hostname);
+ Instance instance = HdfsZooInstance.getInstance();
+ ServerConfiguration conf = new ServerConfiguration(instance);
+ Accumulo.init(fs, conf, "master");
+ Master master = new Master(conf, fs, hostname);
Accumulo.enableTracing(hostname, "master");
master.run();
} catch (Exception ex) {
@@ -2157,7 +2166,7 @@ public class Master implements LiveTServ
@Override
public void newLogger(String address) {
try {
- RemoteLogger remote = new RemoteLogger(address);
+ RemoteLogger remote = new RemoteLogger(address, getSystemConfiguration());
for (String onDisk : remote.getClosedLogs()) {
Path path = new Path(ServerConstants.getRecoveryDir(), onDisk + ".failed");
if (fs.exists(path)) {
@@ -2301,4 +2310,12 @@ public class Master implements LiveTServ
return this.instance;
}
+ public AccumuloConfiguration getSystemConfiguration() {
+ return serverConfig.getConfiguration();
+ }
+
+ public ServerConfiguration getConfiguration() {
+ return serverConfig;
+ }
+
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java Fri Apr 6 16:35:29 2012
@@ -30,6 +30,7 @@ import org.apache.accumulo.core.master.t
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.thrift.TException;
@@ -139,4 +140,13 @@ public class ChaoticLoadBalancer extends
return 100;
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.accumulo.server.master.balancer.TabletBalancer#init(org.apache.accumulo.server.conf.ServerConfiguration)
+ */
+ @Override
+ public void init(ServerConfiguration conf) {
+ }
+
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/DefaultLoadBalancer.java Fri Apr 6 16:35:29 2012
@@ -22,14 +22,15 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
-import java.util.Map.Entry;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.log4j.Logger;
@@ -308,4 +309,13 @@ public class DefaultLoadBalancer extends
}
return 5 * 1000;
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.accumulo.server.master.balancer.TabletBalancer#init(org.apache.accumulo.server.conf.ServerConfiguration)
+ */
+ @Override
+ public void init(ServerConfiguration conf) {
+ }
}
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java Fri Apr 6 16:35:29 2012
@@ -44,6 +44,11 @@ public class TableLoadBalancer extends T
private static final Logger log = Logger.getLogger(TableLoadBalancer.class);
Map<String,TabletBalancer> perTableBalancers = new HashMap<String,TabletBalancer>();
+ ServerConfiguration config;
+
+ public void init(ServerConfiguration config) {
+ this.config = config;
+ }
private TabletBalancer constructNewBalancerForTable(String clazzName, String table) throws Exception {
Class<? extends TabletBalancer> clazz = AccumuloClassLoader.loadClass(clazzName, TabletBalancer.class);
@@ -52,7 +57,7 @@ public class TableLoadBalancer extends T
}
protected String getLoadBalancerClassNameForTable(String table) {
- return ServerConfiguration.getTableConfiguration(table).get(Property.TABLE_LOAD_BALANCER);
+ return config.getTableConfiguration(table).get(Property.TABLE_LOAD_BALANCER);
}
protected TabletBalancer getBalancerForTable(String table) {
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java Fri Apr 6 16:35:29 2012
@@ -26,8 +26,8 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
@@ -41,6 +41,15 @@ public abstract class TabletBalancer {
private static final Logger log = Logger.getLogger(TabletBalancer.class);
+ protected ServerConfiguration configuration;
+
+ /**
+ * Initialize the TabletBalancer. This gives the balancer the opportunity to read the configuration.
+ */
+ public void init(ServerConfiguration conf) {
+ configuration = conf;
+ }
+
/**
* Assign tablets to tablet servers. This method is called whenever the master finds tablets that are unassigned.
*
@@ -86,7 +95,7 @@ public abstract class TabletBalancer {
*/
public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
log.debug("Scanning tablet server " + tserver + " for table " + tableId);
- Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), ServerConfiguration.getSystemConfiguration());
+ Iface client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), tserver.getLocation(), configuration.getConfiguration());
try {
List<TabletStats> onlineTabletsForTable = client.getTabletStats(null, SecurityConstants.getSystemCredentials(), tableId);
return onlineTabletsForTable;
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/TabletStateStore.java Fri Apr 6 16:35:29 2012
@@ -20,8 +20,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
-import org.apache.accumulo.core.Constants;
-
/**
* Interface for storing information about tablet assignments. There are three implementations:
*
@@ -68,7 +66,7 @@ public abstract class TabletStateStore i
public static void unassign(TabletLocationState tls) throws DistributedStoreException {
TabletStateStore store;
- if (tls.extent.equals(Constants.ROOT_TABLET_EXTENT)) {
+ if (tls.extent.isRootTablet()) {
store = new ZooTabletStateStore();
} else {
store = new MetaDataStateStore();
@@ -78,7 +76,7 @@ public abstract class TabletStateStore i
public static void setLocation(Assignment assignment) throws DistributedStoreException {
TabletStateStore store;
- if (assignment.tablet.equals(Constants.ROOT_TABLET_EXTENT)) {
+ if (assignment.tablet.isRootTablet()) {
store = new ZooTabletStateStore();
} else {
store = new MetaDataStateStore();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java Fri Apr 6 16:35:29 2012
@@ -337,24 +337,10 @@ class CompleteBulkImport extends MasterR
class LoadFiles extends MasterRepo {
private static final long serialVersionUID = 1L;
- final static int THREAD_POOL_SIZE = ServerConfiguration.getSystemConfiguration().getCount(Property.MASTER_BULK_THREADPOOL_SIZE);
private static ExecutorService threadPool = null;
static {
- if (threadPool == null) {
- ThreadFactory threadFactory = new ThreadFactory() {
- int count = 0;
-
- @Override
- public Thread newThread(Runnable r) {
- return new Daemon(r, "bulk loader " + count++);
- }
- };
- ThreadPoolExecutor pool = new ThreadPoolExecutor(THREAD_POOL_SIZE, THREAD_POOL_SIZE, 1l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
- threadFactory);
- pool.allowCoreThreadTimeOut(true);
- threadPool = new TraceExecutorService(pool);
- }
+
}
private static final Logger log = Logger.getLogger(BulkImport.class);
@@ -379,6 +365,24 @@ class LoadFiles extends MasterRepo {
return 0;
}
+ synchronized void initializeThreadPool(Master master) {
+ if (threadPool == null) {
+ int THREAD_POOL_SIZE = master.getSystemConfiguration().getCount(Property.MASTER_BULK_THREADPOOL_SIZE);
+ ThreadFactory threadFactory = new ThreadFactory() {
+ int count = 0;
+
+ @Override
+ public Thread newThread(Runnable r) {
+ return new Daemon(r, "bulk loader " + count++);
+ }
+ };
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(THREAD_POOL_SIZE, THREAD_POOL_SIZE, 1l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ threadFactory);
+ pool.allowCoreThreadTimeOut(true);
+ threadPool = new TraceExecutorService(pool);
+ }
+ }
+
@Override
public Repo<Master> call(final long tid, final Master master) throws Exception {
final SiteConfiguration conf = ServerConfiguration.getSiteConfiguration();
Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java?rev=1310457&r1=1310456&r2=1310457&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tserverOps/DisconnectLogger.java Fri Apr 6 16:35:29 2012
@@ -26,7 +26,6 @@ import org.apache.accumulo.core.util.Add
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil.NodeMissingPolicy;
-import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fate.Repo;
import org.apache.accumulo.server.master.Master;
import org.apache.accumulo.server.master.tableOps.MasterRepo;
@@ -60,7 +59,7 @@ public class DisconnectLogger extends Ma
InetSocketAddress addr = AddressUtil.parseAddress(entry.getValue(), 0);
if (addr.getAddress().getHostAddress().equals(location)) {
foundMatch = true;
- MutationLogger.Iface client = ThriftUtil.getClient(new MutationLogger.Client.Factory(), addr, ServerConfiguration.getSystemConfiguration());
+ MutationLogger.Iface client = ThriftUtil.getClient(new MutationLogger.Client.Factory(), addr, m.getSystemConfiguration());
try {
client.beginShutdown(null, SecurityConstants.getSystemCredentials());
} catch (Exception ex) {