You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/05/24 21:14:06 UTC
svn commit: r1342397 - in /accumulo/branches/ACCUMULO-578:
core/src/main/java/org/apache/accumulo/core/conf/
core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/
core/src/main/java/org/apache/accumulo/core/util/ core/src/main/thrift/
server...
Author: ecn
Date: Thu May 24 19:14:05 2012
New Revision: 1342397
URL: http://svn.apache.org/viewvc?rev=1342397&view=rev
Log:
ACCUMULO-578 added back gc
Modified:
accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java Thu May 24 19:14:05 2012
@@ -152,27 +152,12 @@ public enum Property {
"The replication to use when writing the Write-Ahead log to HDFS. If zero, it will use the HDFS default replication setting."),
TSERV_RECOVERY_MAX_CONCURRENT("tserver.recovery.concurrent.max", "2", PropertyType.COUNT, "The maximum number of threads to use to sort logs during recovery"),
TSERV_SORT_BUFFER_SIZE("tserver.sort.buffer.size", "200M", PropertyType.MEMORY, "The amount of memory to use when sorting logs during recovery."),
+ TSERV_ARCHIVE_WALOGS("tserver.archive.walogs", "false", PropertyType.BOOLEAN, "Keep copies of the WALOGs for debugging purposes"),
// properties that are specific to logger server behavior
LOGGER_PREFIX("logger.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the write-ahead logger servers"),
- LOGGER_PORT("logger.port.client", "11224", PropertyType.PORT, "The port used for write-ahead logger services"),
- LOGGER_COPY_THREADPOOL_SIZE("logger.copy.threadpool.size", "2", PropertyType.COUNT,
- "size of the thread pool used to copy files from the local log area to HDFS"),
LOGGER_DIR("logger.dir.walog", "walogs", PropertyType.PATH,
"The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories."),
- LOGGER_PORTSEARCH("logger.port.search", "false", PropertyType.BOOLEAN, "if the port above is in use, search higher ports until one is available"),
- LOGGER_ARCHIVE("logger.archive", "false", PropertyType.BOOLEAN, "determines if logs are archived in hdfs"),
- LOGGER_ARCHIVE_REPLICATION("logger.archive.replication", "0", PropertyType.COUNT,
- "determines the replication factor for walogs archived in hdfs, set to zero to use default"),
- LOGGER_MONITOR_FS(
- "logger.monitor.fs",
- "true",
- PropertyType.BOOLEAN,
- "When enabled the logger will monitor file systems and kill itself when one switches from rw to ro. This is usually and indication that Linux has detected a bad disk."),
- LOGGER_RECOVERY_FILE_REPLICATION("logger.recovery.file.replication", "1", PropertyType.COUNT,
- "When a logger puts a WALOG into HDFS, it will use this as the replication factor."),
- LOGGER_MINTHREADS("logger.server.threads.minimum", "2", PropertyType.COUNT, "The miniumum number of threads to use to handle incoming requests."),
- LOGGER_THREADCHECK("logger.server.threadcheck.time", "1s", PropertyType.TIMEDURATION, "The time between adjustments of the server thread pool."),
// accumulo garbage collector properties
GC_PREFIX("gc.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the accumulo garbage collector."),
@@ -345,8 +330,7 @@ public enum Property {
}
private static final EnumSet<Property> fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED,
- Property.TSERV_SCAN_MAX_OPENFILES, Property.TSERV_LOGGER_COUNT,
- Property.LOGGER_PORT, Property.MASTER_CLIENTPORT, Property.GC_PORT);
+ Property.TSERV_SCAN_MAX_OPENFILES, Property.TSERV_LOGGER_COUNT, Property.MASTER_CLIENTPORT, Property.GC_PORT);
public static boolean isFixedZooPropertyKey(Property key) {
return fixedProperties.contains(key);
Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java Thu May 24 19:14:05 2012
@@ -76,6 +76,8 @@ import org.slf4j.LoggerFactory;
public double sortLog(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, String lock, String path) throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException;
+ public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames) throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException;
+
}
public interface AsyncIface extends org.apache.accumulo.core.client.impl.thrift.ClientService .AsyncIface {
@@ -132,6 +134,8 @@ import org.slf4j.LoggerFactory;
public void sortLog(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, String lock, String path, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.sortLog_call> resultHandler) throws org.apache.thrift.TException;
+ public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.removeLogs_call> resultHandler) throws org.apache.thrift.TException;
+
}
public static class Client extends org.apache.accumulo.core.client.impl.thrift.ClientService.Client implements org.apache.thrift.TServiceClient, Iface {
@@ -1011,6 +1015,44 @@ import org.slf4j.LoggerFactory;
throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "sortLog failed: unknown result");
}
+ public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames) throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException
+ {
+ send_removeLogs(tinfo, credentials, filenames);
+ recv_removeLogs();
+ }
+
+ public void send_removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames) throws org.apache.thrift.TException
+ {
+ oprot_.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.CALL, ++seqid_));
+ removeLogs_args args = new removeLogs_args();
+ args.setTinfo(tinfo);
+ args.setCredentials(credentials);
+ args.setFilenames(filenames);
+ args.write(oprot_);
+ oprot_.writeMessageEnd();
+ oprot_.getTransport().flush();
+ }
+
+ public void recv_removeLogs() throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException
+ {
+ org.apache.thrift.protocol.TMessage msg = iprot_.readMessageBegin();
+ if (msg.type == org.apache.thrift.protocol.TMessageType.EXCEPTION) {
+ org.apache.thrift.TApplicationException x = org.apache.thrift.TApplicationException.read(iprot_);
+ iprot_.readMessageEnd();
+ throw x;
+ }
+ if (msg.seqid != seqid_) {
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.BAD_SEQUENCE_ID, "removeLogs failed: out of sequence response");
+ }
+ removeLogs_result result = new removeLogs_result();
+ result.read(iprot_);
+ iprot_.readMessageEnd();
+ if (result.sec != null) {
+ throw result.sec;
+ }
+ return;
+ }
+
}
public static class AsyncClient extends org.apache.accumulo.core.client.impl.thrift.ClientService.AsyncClient implements AsyncIface {
public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
@@ -2070,6 +2112,44 @@ import org.slf4j.LoggerFactory;
}
}
+ public void removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames, org.apache.thrift.async.AsyncMethodCallback<removeLogs_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ removeLogs_call method_call = new removeLogs_call(tinfo, credentials, filenames, resultHandler, this, protocolFactory, transport);
+ this.currentMethod = method_call;
+ manager.call(method_call);
+ }
+
+ public static class removeLogs_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private org.apache.accumulo.cloudtrace.thrift.TInfo tinfo;
+ private org.apache.accumulo.core.security.thrift.AuthInfo credentials;
+ private List<String> filenames;
+ public removeLogs_call(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.AuthInfo credentials, List<String> filenames, org.apache.thrift.async.AsyncMethodCallback<removeLogs_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.tinfo = tinfo;
+ this.credentials = credentials;
+ this.filenames = filenames;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ removeLogs_args args = new removeLogs_args();
+ args.setTinfo(tinfo);
+ args.setCredentials(credentials);
+ args.setFilenames(filenames);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public void getResult() throws org.apache.accumulo.core.security.thrift.ThriftSecurityException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ (new Client(prot)).recv_removeLogs();
+ }
+ }
+
}
public static class Processor extends org.apache.accumulo.core.client.impl.thrift.ClientService.Processor implements org.apache.thrift.TProcessor {
@@ -2104,6 +2184,7 @@ import org.slf4j.LoggerFactory;
processMap_.put("fastHalt", new fastHalt());
processMap_.put("getActiveScans", new getActiveScans());
processMap_.put("sortLog", new sortLog());
+ processMap_.put("removeLogs", new removeLogs());
}
private Iface iface_;
@@ -2960,6 +3041,44 @@ import org.slf4j.LoggerFactory;
}
+ private class removeLogs implements ProcessFunction {
+ public void process(int seqid, org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException
+ {
+ removeLogs_args args = new removeLogs_args();
+ try {
+ args.read(iprot);
+ } catch (org.apache.thrift.protocol.TProtocolException e) {
+ iprot.readMessageEnd();
+ org.apache.thrift.TApplicationException x = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.PROTOCOL_ERROR, e.getMessage());
+ oprot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.EXCEPTION, seqid));
+ x.write(oprot);
+ oprot.writeMessageEnd();
+ oprot.getTransport().flush();
+ return;
+ }
+ iprot.readMessageEnd();
+ removeLogs_result result = new removeLogs_result();
+ try {
+ iface_.removeLogs(args.tinfo, args.credentials, args.filenames);
+ } catch (org.apache.accumulo.core.security.thrift.ThriftSecurityException sec) {
+ result.sec = sec;
+ } catch (Throwable th) {
+ LOGGER.error("Internal error processing removeLogs", th);
+ org.apache.thrift.TApplicationException x = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, "Internal error processing removeLogs");
+ oprot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.EXCEPTION, seqid));
+ x.write(oprot);
+ oprot.writeMessageEnd();
+ oprot.getTransport().flush();
+ return;
+ }
+ oprot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("removeLogs", org.apache.thrift.protocol.TMessageType.REPLY, seqid));
+ result.write(oprot);
+ oprot.writeMessageEnd();
+ oprot.getTransport().flush();
+ }
+
+ }
+
}
public static class startScan_args implements org.apache.thrift.TBase<startScan_args, startScan_args._Fields>, java.io.Serializable, Cloneable {
@@ -24393,4 +24512,815 @@ import org.slf4j.LoggerFactory;
}
+ public static class removeLogs_args implements org.apache.thrift.TBase<removeLogs_args, removeLogs_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("removeLogs_args");
+
+ private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+ private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+ private static final org.apache.thrift.protocol.TField FILENAMES_FIELD_DESC = new org.apache.thrift.protocol.TField("filenames", org.apache.thrift.protocol.TType.LIST, (short)3);
+
+ public org.apache.accumulo.cloudtrace.thrift.TInfo tinfo;
+ public org.apache.accumulo.core.security.thrift.AuthInfo credentials;
+ public List<String> filenames;
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ TINFO((short)1, "tinfo"),
+ CREDENTIALS((short)2, "credentials"),
+ FILENAMES((short)3, "filenames");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // TINFO
+ return TINFO;
+ case 2: // CREDENTIALS
+ return CREDENTIALS;
+ case 3: // FILENAMES
+ return FILENAMES;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.cloudtrace.thrift.TInfo.class)));
+ tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.security.thrift.AuthInfo.class)));
+ tmpMap.put(_Fields.FILENAMES, new org.apache.thrift.meta_data.FieldMetaData("filenames", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(removeLogs_args.class, metaDataMap);
+ }
+
+ public removeLogs_args() {
+ }
+
+ public removeLogs_args(
+ org.apache.accumulo.cloudtrace.thrift.TInfo tinfo,
+ org.apache.accumulo.core.security.thrift.AuthInfo credentials,
+ List<String> filenames)
+ {
+ this();
+ this.tinfo = tinfo;
+ this.credentials = credentials;
+ this.filenames = filenames;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public removeLogs_args(removeLogs_args other) {
+ if (other.isSetTinfo()) {
+ this.tinfo = new org.apache.accumulo.cloudtrace.thrift.TInfo(other.tinfo);
+ }
+ if (other.isSetCredentials()) {
+ this.credentials = new org.apache.accumulo.core.security.thrift.AuthInfo(other.credentials);
+ }
+ if (other.isSetFilenames()) {
+ List<String> __this__filenames = new ArrayList<String>();
+ for (String other_element : other.filenames) {
+ __this__filenames.add(other_element);
+ }
+ this.filenames = __this__filenames;
+ }
+ }
+
+ public removeLogs_args deepCopy() {
+ return new removeLogs_args(this);
+ }
+
+ @Override
+ public void clear() {
+ this.tinfo = null;
+ this.credentials = null;
+ this.filenames = null;
+ }
+
+ public org.apache.accumulo.cloudtrace.thrift.TInfo getTinfo() {
+ return this.tinfo;
+ }
+
+ public removeLogs_args setTinfo(org.apache.accumulo.cloudtrace.thrift.TInfo tinfo) {
+ this.tinfo = tinfo;
+ return this;
+ }
+
+ public void unsetTinfo() {
+ this.tinfo = null;
+ }
+
+ /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */
+ public boolean isSetTinfo() {
+ return this.tinfo != null;
+ }
+
+ public void setTinfoIsSet(boolean value) {
+ if (!value) {
+ this.tinfo = null;
+ }
+ }
+
+ public org.apache.accumulo.core.security.thrift.AuthInfo getCredentials() {
+ return this.credentials;
+ }
+
+ public removeLogs_args setCredentials(org.apache.accumulo.core.security.thrift.AuthInfo credentials) {
+ this.credentials = credentials;
+ return this;
+ }
+
+ public void unsetCredentials() {
+ this.credentials = null;
+ }
+
+ /** Returns true if field credentials is set (has been assigned a value) and false otherwise */
+ public boolean isSetCredentials() {
+ return this.credentials != null;
+ }
+
+ public void setCredentialsIsSet(boolean value) {
+ if (!value) {
+ this.credentials = null;
+ }
+ }
+
+ public int getFilenamesSize() {
+ return (this.filenames == null) ? 0 : this.filenames.size();
+ }
+
+ public java.util.Iterator<String> getFilenamesIterator() {
+ return (this.filenames == null) ? null : this.filenames.iterator();
+ }
+
+ public void addToFilenames(String elem) {
+ if (this.filenames == null) {
+ this.filenames = new ArrayList<String>();
+ }
+ this.filenames.add(elem);
+ }
+
+ public List<String> getFilenames() {
+ return this.filenames;
+ }
+
+ public removeLogs_args setFilenames(List<String> filenames) {
+ this.filenames = filenames;
+ return this;
+ }
+
+ public void unsetFilenames() {
+ this.filenames = null;
+ }
+
+ /** Returns true if field filenames is set (has been assigned a value) and false otherwise */
+ public boolean isSetFilenames() {
+ return this.filenames != null;
+ }
+
+ public void setFilenamesIsSet(boolean value) {
+ if (!value) {
+ this.filenames = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case TINFO:
+ if (value == null) {
+ unsetTinfo();
+ } else {
+ setTinfo((org.apache.accumulo.cloudtrace.thrift.TInfo)value);
+ }
+ break;
+
+ case CREDENTIALS:
+ if (value == null) {
+ unsetCredentials();
+ } else {
+ setCredentials((org.apache.accumulo.core.security.thrift.AuthInfo)value);
+ }
+ break;
+
+ case FILENAMES:
+ if (value == null) {
+ unsetFilenames();
+ } else {
+ setFilenames((List<String>)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case TINFO:
+ return getTinfo();
+
+ case CREDENTIALS:
+ return getCredentials();
+
+ case FILENAMES:
+ return getFilenames();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case TINFO:
+ return isSetTinfo();
+ case CREDENTIALS:
+ return isSetCredentials();
+ case FILENAMES:
+ return isSetFilenames();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof removeLogs_args)
+ return this.equals((removeLogs_args)that);
+ return false;
+ }
+
+ public boolean equals(removeLogs_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_tinfo = true && this.isSetTinfo();
+ boolean that_present_tinfo = true && that.isSetTinfo();
+ if (this_present_tinfo || that_present_tinfo) {
+ if (!(this_present_tinfo && that_present_tinfo))
+ return false;
+ if (!this.tinfo.equals(that.tinfo))
+ return false;
+ }
+
+ boolean this_present_credentials = true && this.isSetCredentials();
+ boolean that_present_credentials = true && that.isSetCredentials();
+ if (this_present_credentials || that_present_credentials) {
+ if (!(this_present_credentials && that_present_credentials))
+ return false;
+ if (!this.credentials.equals(that.credentials))
+ return false;
+ }
+
+ boolean this_present_filenames = true && this.isSetFilenames();
+ boolean that_present_filenames = true && that.isSetFilenames();
+ if (this_present_filenames || that_present_filenames) {
+ if (!(this_present_filenames && that_present_filenames))
+ return false;
+ if (!this.filenames.equals(that.filenames))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(removeLogs_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ removeLogs_args typedOther = (removeLogs_args)other;
+
+ lastComparison = Boolean.valueOf(isSetTinfo()).compareTo(typedOther.isSetTinfo());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTinfo()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, typedOther.tinfo);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetCredentials()).compareTo(typedOther.isSetCredentials());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetCredentials()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, typedOther.credentials);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetFilenames()).compareTo(typedOther.isSetFilenames());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetFilenames()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.filenames, typedOther.filenames);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField field;
+ iprot.readStructBegin();
+ while (true)
+ {
+ field = iprot.readFieldBegin();
+ if (field.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (field.id) {
+ case 1: // TINFO
+ if (field.type == org.apache.thrift.protocol.TType.STRUCT) {
+ this.tinfo = new org.apache.accumulo.cloudtrace.thrift.TInfo();
+ this.tinfo.read(iprot);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
+ case 2: // CREDENTIALS
+ if (field.type == org.apache.thrift.protocol.TType.STRUCT) {
+ this.credentials = new org.apache.accumulo.core.security.thrift.AuthInfo();
+ this.credentials.read(iprot);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
+ case 3: // FILENAMES
+ if (field.type == org.apache.thrift.protocol.TType.LIST) {
+ {
+ org.apache.thrift.protocol.TList _list109 = iprot.readListBegin();
+ this.filenames = new ArrayList<String>(_list109.size);
+ for (int _i110 = 0; _i110 < _list109.size; ++_i110)
+ {
+ String _elem111;
+ _elem111 = iprot.readString();
+ this.filenames.add(_elem111);
+ }
+ iprot.readListEnd();
+ }
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (this.tinfo != null) {
+ oprot.writeFieldBegin(TINFO_FIELD_DESC);
+ this.tinfo.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ if (this.credentials != null) {
+ oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC);
+ this.credentials.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ if (this.filenames != null) {
+ oprot.writeFieldBegin(FILENAMES_FIELD_DESC);
+ {
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, this.filenames.size()));
+ for (String _iter112 : this.filenames)
+ {
+ oprot.writeString(_iter112);
+ }
+ oprot.writeListEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("removeLogs_args(");
+ boolean first = true;
+
+ sb.append("tinfo:");
+ if (this.tinfo == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.tinfo);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("credentials:");
+ if (this.credentials == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.credentials);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("filenames:");
+ if (this.filenames == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.filenames);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ }
+
+ public static class removeLogs_result implements org.apache.thrift.TBase<removeLogs_result, removeLogs_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("removeLogs_result");
+
+ private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ public org.apache.accumulo.core.security.thrift.ThriftSecurityException sec;
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SEC((short)1, "sec");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // SEC
+ return SEC;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(removeLogs_result.class, metaDataMap);
+ }
+
+ public removeLogs_result() {
+ }
+
+ public removeLogs_result(
+ org.apache.accumulo.core.security.thrift.ThriftSecurityException sec)
+ {
+ this();
+ this.sec = sec;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public removeLogs_result(removeLogs_result other) {
+ if (other.isSetSec()) {
+ this.sec = new org.apache.accumulo.core.security.thrift.ThriftSecurityException(other.sec);
+ }
+ }
+
+ public removeLogs_result deepCopy() {
+ return new removeLogs_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.sec = null;
+ }
+
+ public org.apache.accumulo.core.security.thrift.ThriftSecurityException getSec() {
+ return this.sec;
+ }
+
+ public removeLogs_result setSec(org.apache.accumulo.core.security.thrift.ThriftSecurityException sec) {
+ this.sec = sec;
+ return this;
+ }
+
+ public void unsetSec() {
+ this.sec = null;
+ }
+
+ /** Returns true if field sec is set (has been assigned a value) and false otherwise */
+ public boolean isSetSec() {
+ return this.sec != null;
+ }
+
+ public void setSecIsSet(boolean value) {
+ if (!value) {
+ this.sec = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SEC:
+ if (value == null) {
+ unsetSec();
+ } else {
+ setSec((org.apache.accumulo.core.security.thrift.ThriftSecurityException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SEC:
+ return getSec();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SEC:
+ return isSetSec();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof removeLogs_result)
+ return this.equals((removeLogs_result)that);
+ return false;
+ }
+
+ public boolean equals(removeLogs_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_sec = true && this.isSetSec();
+ boolean that_present_sec = true && that.isSetSec();
+ if (this_present_sec || that_present_sec) {
+ if (!(this_present_sec && that_present_sec))
+ return false;
+ if (!this.sec.equals(that.sec))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(removeLogs_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ removeLogs_result typedOther = (removeLogs_result)other;
+
+ lastComparison = Boolean.valueOf(isSetSec()).compareTo(typedOther.isSetSec());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSec()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, typedOther.sec);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField field;
+ iprot.readStructBegin();
+ while (true)
+ {
+ field = iprot.readFieldBegin();
+ if (field.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (field.id) {
+ case 1: // SEC
+ if (field.type == org.apache.thrift.protocol.TType.STRUCT) {
+ this.sec = new org.apache.accumulo.core.security.thrift.ThriftSecurityException();
+ this.sec.read(iprot);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ oprot.writeStructBegin(STRUCT_DESC);
+
+ if (this.isSetSec()) {
+ oprot.writeFieldBegin(SEC_FIELD_DESC);
+ this.sec.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("removeLogs_result(");
+ boolean first = true;
+
+ sb.append("sec:");
+ if (this.sec == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.sec);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ }
+
}
Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java Thu May 24 19:14:05 2012
@@ -23,7 +23,9 @@ import org.apache.thrift.transport.TSock
public class AddressUtil {
static public InetSocketAddress parseAddress(String address, int defaultPort) throws NumberFormatException {
- final String[] parts = address.split(":", 2);
+ String[] parts = address.split(":", 2);
+ if (address.contains("+"))
+ parts = address.split(":", 2);
if (parts.length == 2) {
if (parts[1].isEmpty())
return new InetSocketAddress(parts[0], defaultPort);
Modified: accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift (original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/thrift/tabletserver.thrift Thu May 24 19:14:05 2012
@@ -147,6 +147,7 @@ service TabletClientService extends clie
list<ActiveScan> getActiveScans(2:cloudtrace.TInfo tinfo, 1:security.AuthInfo credentials) throws (1:security.ThriftSecurityException sec)
double sortLog(1:cloudtrace.TInfo tinfo, 2:security.AuthInfo credentials, 3:string lock, 4:string path) throws (1:security.ThriftSecurityException sec);
+ oneway void removeLogs(1:cloudtrace.TInfo tinfo, 2:security.AuthInfo credentials, 3:list<string> filenames) throws (1:security.ThriftSecurityException sec);
}
typedef i32 TabletID
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java Thu May 24 19:14:05 2012
@@ -17,22 +17,34 @@
package org.apache.accumulo.server.gc;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.UUID;
import org.apache.accumulo.cloudtrace.instrument.Span;
import org.apache.accumulo.cloudtrace.instrument.Trace;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
+import org.apache.accumulo.core.security.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
+import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.util.AddressUtil;
import org.apache.accumulo.server.util.MetadataTable;
import org.apache.accumulo.server.util.MetadataTable.LogEntry;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException;
@@ -93,6 +105,33 @@ public class GarbageCollectWriteAheadLog
}
private int removeFiles(Map<String,ArrayList<String>> serverToFileMap, final GCStatus status) {
+ for (Entry<String,ArrayList<String>> entry : serverToFileMap.entrySet()) {
+ if (entry.getKey().length() == 0) {
+ // old-style log entry, just remove it
+ for (String filename : entry.getValue()) {
+ log.debug("Removing old-style WAL " + entry.getValue());
+ try {
+ fs.delete(new Path(Constants.getWalDirectory(conf), filename), true);
+ } catch (IOException ex) {
+ log.error("Unable to delete wal " + filename + ": " + ex);
+ }
+ }
+ } else {
+ InetSocketAddress address = AddressUtil.parseAddress(entry.getKey(), Property.TSERV_CLIENTPORT);
+ Iface tserver = null;
+ try {
+ tserver = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, conf);
+ tserver.removeLogs(null, SecurityConstants.getSystemCredentials(), entry.getValue());
+ } catch (ThriftSecurityException e) {
+ log.warn("Unexpected security exception during log file cleanup while talking to " + address + ". Is your configuration consistent over all nodes?");
+ } catch (TException e) {
+ log.warn("Error talking to " + address + ": " + e);
+ } finally {
+ if (tserver != null)
+ ThriftUtil.returnClient(tserver);
+ }
+ }
+ }
return 0;
}
@@ -124,8 +163,40 @@ public class GarbageCollectWriteAheadLog
}
private int scanServers(Map<String,String> fileToServerMap) throws Exception {
+ Path walRoot = new Path(Constants.getWalDirectory(conf));
+ for (FileStatus status : fs.listStatus(walRoot)) {
+ String name = status.getPath().getName();
+ if (status.isDir()) {
+ for (FileStatus file : fs.listStatus(new Path(walRoot, name))) {
+ if (isUUID(file.getPath().getName()))
+ fileToServerMap.put(file.getPath().getName(), name);
+ else {
+ log.info("Ignoring file " + file.getPath() + " because it doesn't look like a uuid");
+ }
+ }
+ } else if (isUUID(name)) {
+ // old-style WAL are not under a directory
+ fileToServerMap.put(name, "");
+ } else {
+ log.info("Ignoring file " + name + " because it doesn't look like a uuid");
+ }
+ }
+
int count = 0;
return count;
}
+ /**
+ * @param name
+ * @return
+ */
+ static private boolean isUUID(String name) {
+ try {
+ UUID.fromString(name);
+ return true;
+ } catch (IllegalArgumentException ex) {
+ return false;
+ }
+ }
+
}
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java Thu May 24 19:14:05 2012
@@ -78,10 +78,6 @@ public class CoordinateRecoveryTask impl
String failedFileName() {
return fullName(file + "/failed");
}
-
- public String unsortedFileName() {
- return fullName(file);
- }
}
interface JobComplete {
@@ -98,7 +94,6 @@ public class CoordinateRecoveryTask impl
private class RecoveryJob {
final LogFile logFile;
final long copyStartTime;
- double copySize = 0;
JobComplete notify = null;
final AccumuloConfiguration config;
@@ -114,9 +109,11 @@ public class CoordinateRecoveryTask impl
try {
// Ask the logging server to put the file in HDFS
IRemoteLogger logger = new DfsLogger(conf);
- String source = Constants.getWalDirectory(config) + "/" + logFile.file;
+ String source = Constants.getWalDirectory(config) + "/" + logFile.server + "/" + logFile.file;
+ if (logFile.server.contains(":"))
+ source = Constants.getWalDirectory(config) + "/" + logFile.file;
String dest = Constants.getRecoveryDir(config) + "/" + logFile.file;
- copySize = logger.startCopy(source, dest);
+ logger.startCopy(source, dest);
} catch (Throwable t) {
log.warn("Unable to recover " + logFile + "(" + t + ")", t);
fail();
@@ -166,14 +163,13 @@ public class CoordinateRecoveryTask impl
}
synchronized public String toString() {
- return String.format("Copying %s from %s (for %f seconds) %2.1f", logFile.file, logFile.server, elapsedMillis() / 1000., copiedSoFar() * 100. / copySize);
+ return String.format("Copying %s from %s (for %f seconds) %2.1f", logFile.file, logFile.server, elapsedMillis() / 1000., copiedSoFar() * 100.);
}
- synchronized long copiedSoFar() {
+ synchronized double copiedSoFar() {
try {
ContentSummary contentSummary = fs.getContentSummary(new Path(logFile.recoveryFileName()));
- // map files are bigger than sequence files
- return (long) (contentSummary.getSpaceConsumed() * .8);
+ return contentSummary.getSpaceConsumed() / (double) config.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE);
} catch (Exception ex) {
return 0;
}
@@ -181,7 +177,7 @@ public class CoordinateRecoveryTask impl
synchronized public RecoveryStatus getStatus() throws IOException {
try {
- return new RecoveryStatus(logFile.server, logFile.file, 0., 0., (int) (System.currentTimeMillis() - copyStartTime), (copiedSoFar() / (double) copySize));
+ return new RecoveryStatus(logFile.server, logFile.file, 0., 0., (int) (System.currentTimeMillis() - copyStartTime), copiedSoFar());
} catch (Exception e) {
return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
}
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java Thu May 24 19:14:05 2012
@@ -110,6 +110,7 @@ import org.apache.accumulo.server.tablet
import org.apache.accumulo.server.tabletserver.TabletServer.TservConstraintEnv;
import org.apache.accumulo.server.tabletserver.TabletServerResourceManager.TabletResourceManager;
import org.apache.accumulo.server.tabletserver.TabletStatsKeeper.Operation;
+import org.apache.accumulo.server.tabletserver.log.DfsLogger;
import org.apache.accumulo.server.tabletserver.log.IRemoteLogger;
import org.apache.accumulo.server.tabletserver.log.MutationReceiver;
import org.apache.accumulo.server.tabletserver.mastermessage.TabletStatusMessage;
@@ -1456,6 +1457,15 @@ public class Tablet {
throw new RuntimeException(t);
}
}
+ // make some closed references that represent the recovered logs
+ currentLogs = new HashSet<IRemoteLogger>();
+ for (LogEntry logEntry : logEntries) {
+ for (String log : logEntry.logSet) {
+ String[] parts = log.split("/", 2);
+ currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), parts[1]));
+ }
+ }
+
log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries()
+ " entries created)");
}
@@ -3606,6 +3616,14 @@ public class Tablet {
private Set<IRemoteLogger> currentLogs = new HashSet<IRemoteLogger>();
+ synchronized public Set<String> getCurrentLogs() {
+ Set<String> result = new HashSet<String>();
+ for (IRemoteLogger log : currentLogs) {
+ result.add(log.toString());
+ }
+ return result;
+ }
+
private Set<String> beginClearingUnusedLogs() {
Set<String> doomed = new HashSet<String>();
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java Thu May 24 19:14:05 2012
@@ -2074,6 +2074,49 @@ public class TabletServer extends Abstra
}
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo,
+ * org.apache.accumulo.core.security.thrift.AuthInfo, java.util.List)
+ */
+ @Override
+ public void removeLogs(TInfo tinfo, AuthInfo credentials, List<String> filenames) throws ThriftSecurityException, TException {
+ Path logDir = new Path(Constants.getWalDirectory(acuConf), getClientAddressString());
+ Set<String> loggers = new HashSet<String>();
+ logger.getLoggers(loggers);
+ nextFile:
+ for (String filename : filenames) {
+ for (String logger : loggers) {
+ if (logger.contains(filename))
+ continue nextFile;
+ }
+ // this check is not strictly necessary
+ synchronized (onlineTablets) {
+ for (Entry<KeyExtent,Tablet> entry : onlineTablets.entrySet()) {
+ for (String current : entry.getValue().getCurrentLogs()) {
+ if (current.contains(filename)) {
+ log.error("Attempting to remove a write-ahead log that is in use. This should never happen!");
+ log.info("Attempted to delete " + filename + " from tablet " + entry.getKey());
+ continue nextFile;
+ }
+ }
+ }
+ }
+ try {
+ if (acuConf.getBoolean(Property.TSERV_ARCHIVE_WALOGS)) {
+ log.info("Archiving walog " + filename);
+ fs.rename(new Path(logDir, filename), new Path(Constants.getBaseDir(acuConf) + "/walogArchive", filename));
+ } else {
+ log.info("Deleting walog " + filename);
+ fs.delete(new Path(logDir, filename), true);
+ }
+ } catch (IOException e) {
+ log.warn("Error attempting to delete write-ahead log " + filename + ": " + e);
+ }
+ }
+ }
+
}
private class SplitRunner implements Runnable {
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java Thu May 24 19:14:05 2012
@@ -24,6 +24,7 @@ import static org.apache.accumulo.server
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Random;
@@ -40,6 +41,7 @@ import org.apache.accumulo.core.data.Mut
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface;
import org.apache.accumulo.core.tabletserver.thrift.TabletMutations;
import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.server.logger.LogFileKey;
import org.apache.accumulo.server.logger.LogFileValue;
@@ -186,6 +188,7 @@ public class DfsLogger implements IRemot
private ServerConfig conf;
private FSDataOutputStream logFile;
private Path logPath;
+ private String logger;
public DfsLogger(ServerConfig conf) throws IOException {
this.conf = conf;
@@ -196,10 +199,11 @@ public class DfsLogger implements IRemot
this.logPath = new Path(Constants.getWalDirectory(conf.getConfiguration()), filename);
}
- public synchronized void open() throws IOException {
+ public synchronized void open(String address) throws IOException {
String filename = UUID.randomUUID().toString();
+ logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
- logPath = new Path(Constants.getWalDirectory(conf.getConfiguration()), filename);
+ logPath = new Path(Constants.getWalDirectory(conf.getConfiguration()) + "/" + logger + "/" + filename);
try {
FileSystem fs = conf.getFileSystem();
short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
@@ -243,7 +247,7 @@ public class DfsLogger implements IRemot
*/
@Override
public String getLogger() {
- return "";
+ return logger;
}
/* (non-Javadoc)
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/IRemoteLogger.java Thu May 24 19:14:05 2012
@@ -57,6 +57,6 @@ public interface IRemoteLogger {
public abstract void removeFile(List<String> files) throws IOException;
- public abstract void open() throws IOException;
+ public abstract void open(String address) throws IOException;
}
\ No newline at end of file
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/log/TabletServerLogger.java Thu May 24 19:14:05 2012
@@ -182,7 +182,7 @@ public class TabletServerLogger {
try {
DfsLogger alog = new DfsLogger(tserver.getServerConfig());
- alog.open();
+ alog.open(tserver.getClientAddressString());
loggers.add(alog);
logSetId.incrementAndGet();
return;
Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java?rev=1342397&r1=1342396&r2=1342397&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java (original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/performance/thrift/NullTserver.java Thu May 24 19:14:05 2012
@@ -197,9 +197,18 @@ public class NullTserver {
*/
@Override
public double sortLog(TInfo tinfo, AuthInfo credentials, String lock, String path) throws ThriftSecurityException, TException {
- // TODO Auto-generated method stub
return 0;
}
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Iface#removeLogs(org.apache.accumulo.cloudtrace.thrift.TInfo,
+ * org.apache.accumulo.core.security.thrift.AuthInfo, java.util.List)
+ */
+ @Override
+ public void removeLogs(TInfo tinfo, AuthInfo credentials, List<String> filenames) throws ThriftSecurityException, TException {
+ }
}
public static void main(String[] args) throws Exception {