You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by GitBox <gi...@apache.org> on 2019/01/10 15:40:30 UTC
[GitHub] keith-turner closed pull request #871: Check transaction are alive
for new bulk import RPCs
keith-turner closed pull request #871: Check transaction are alive for new bulk import RPCs
URL: https://github.com/apache/accumulo/pull/871
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
index ce61943477..a14e94dc61 100644
--- a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
+++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
@@ -58,7 +58,7 @@
public java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent> bulkImport(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException;
- public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent, java.lang.String dir, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files, boolean setTime) throws org.apache.thrift.TException;
+ public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.lang.String dir, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime) throws org.apache.thrift.TException;
public void splitTablet(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.nio.ByteBuffer splitPoint) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, NotServingTabletException, org.apache.thrift.TException;
@@ -134,7 +134,7 @@
public void bulkImport(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<java.util.List<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent>> resultHandler) throws org.apache.thrift.TException;
- public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent, java.lang.String dir, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException;
+ public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.lang.String dir, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException;
public void splitTablet(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.nio.ByteBuffer splitPoint, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException;
@@ -618,18 +618,17 @@ public void send_bulkImport(org.apache.accumulo.core.trace.thrift.TInfo tinfo, o
throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "bulkImport failed: unknown result");
}
- public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent, java.lang.String dir, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files, boolean setTime) throws org.apache.thrift.TException
+ public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.lang.String dir, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime) throws org.apache.thrift.TException
{
- send_loadFiles(tinfo, credentials, tid, keyExtent, dir, files, setTime);
+ send_loadFiles(tinfo, credentials, tid, dir, files, setTime);
}
- public void send_loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent, java.lang.String dir, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files, boolean setTime) throws org.apache.thrift.TException
+ public void send_loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.lang.String dir, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime) throws org.apache.thrift.TException
{
loadFiles_args args = new loadFiles_args();
args.setTinfo(tinfo);
args.setCredentials(credentials);
args.setTid(tid);
- args.setKeyExtent(keyExtent);
args.setDir(dir);
args.setFiles(files);
args.setSetTime(setTime);
@@ -1751,9 +1750,9 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa
}
}
- public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent, java.lang.String dir, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException {
+ public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.lang.String dir, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException {
checkReady();
- loadFiles_call method_call = new loadFiles_call(tinfo, credentials, tid, keyExtent, dir, files, setTime, resultHandler, this, ___protocolFactory, ___transport);
+ loadFiles_call method_call = new loadFiles_call(tinfo, credentials, tid, dir, files, setTime, resultHandler, this, ___protocolFactory, ___transport);
this.___currentMethod = method_call;
___manager.call(method_call);
}
@@ -1762,16 +1761,14 @@ public void loadFiles(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apa
private org.apache.accumulo.core.trace.thrift.TInfo tinfo;
private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials;
private long tid;
- private org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent;
private java.lang.String dir;
- private java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files;
+ private java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files;
private boolean setTime;
- public loadFiles_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent, java.lang.String dir, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ public loadFiles_call(org.apache.accumulo.core.trace.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long tid, java.lang.String dir, java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files, boolean setTime, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
super(client, protocolFactory, transport, resultHandler, true);
this.tinfo = tinfo;
this.credentials = credentials;
this.tid = tid;
- this.keyExtent = keyExtent;
this.dir = dir;
this.files = files;
this.setTime = setTime;
@@ -1783,7 +1780,6 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa
args.setTinfo(tinfo);
args.setCredentials(credentials);
args.setTid(tid);
- args.setKeyExtent(keyExtent);
args.setDir(dir);
args.setFiles(files);
args.setSetTime(setTime);
@@ -3099,7 +3095,7 @@ protected boolean handleRuntimeExceptions() {
}
public org.apache.thrift.TBase getResult(I iface, loadFiles_args args) throws org.apache.thrift.TException {
- iface.loadFiles(args.tinfo, args.credentials, args.tid, args.keyExtent, args.dir, args.files, args.setTime);
+ iface.loadFiles(args.tinfo, args.credentials, args.tid, args.dir, args.files, args.setTime);
return null;
}
}
@@ -4642,7 +4638,7 @@ protected boolean isOneway() {
}
public void start(I iface, loadFiles_args args, org.apache.thrift.async.AsyncMethodCallback<Void> resultHandler) throws org.apache.thrift.TException {
- iface.loadFiles(args.tinfo, args.credentials, args.tid, args.keyExtent, args.dir, args.files, args.setTime,resultHandler);
+ iface.loadFiles(args.tinfo, args.credentials, args.tid, args.dir, args.files, args.setTime,resultHandler);
}
}
@@ -24322,10 +24318,9 @@ public void read(org.apache.thrift.protocol.TProtocol prot, bulkImport_result st
private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1);
private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2);
private static final org.apache.thrift.protocol.TField TID_FIELD_DESC = new org.apache.thrift.protocol.TField("tid", org.apache.thrift.protocol.TType.I64, (short)3);
- private static final org.apache.thrift.protocol.TField KEY_EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("keyExtent", org.apache.thrift.protocol.TType.STRUCT, (short)4);
- private static final org.apache.thrift.protocol.TField DIR_FIELD_DESC = new org.apache.thrift.protocol.TField("dir", org.apache.thrift.protocol.TType.STRING, (short)5);
- private static final org.apache.thrift.protocol.TField FILES_FIELD_DESC = new org.apache.thrift.protocol.TField("files", org.apache.thrift.protocol.TType.MAP, (short)6);
- private static final org.apache.thrift.protocol.TField SET_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("setTime", org.apache.thrift.protocol.TType.BOOL, (short)7);
+ private static final org.apache.thrift.protocol.TField DIR_FIELD_DESC = new org.apache.thrift.protocol.TField("dir", org.apache.thrift.protocol.TType.STRING, (short)4);
+ private static final org.apache.thrift.protocol.TField FILES_FIELD_DESC = new org.apache.thrift.protocol.TField("files", org.apache.thrift.protocol.TType.MAP, (short)5);
+ private static final org.apache.thrift.protocol.TField SET_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("setTime", org.apache.thrift.protocol.TType.BOOL, (short)6);
private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new loadFiles_argsStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new loadFiles_argsTupleSchemeFactory();
@@ -24333,9 +24328,8 @@ public void read(org.apache.thrift.protocol.TProtocol prot, bulkImport_result st
public org.apache.accumulo.core.trace.thrift.TInfo tinfo; // required
public org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required
public long tid; // required
- public org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent; // required
public java.lang.String dir; // required
- public java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files; // required
+ public java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files; // required
public boolean setTime; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
@@ -24343,10 +24337,9 @@ public void read(org.apache.thrift.protocol.TProtocol prot, bulkImport_result st
TINFO((short)1, "tinfo"),
CREDENTIALS((short)2, "credentials"),
TID((short)3, "tid"),
- KEY_EXTENT((short)4, "keyExtent"),
- DIR((short)5, "dir"),
- FILES((short)6, "files"),
- SET_TIME((short)7, "setTime");
+ DIR((short)4, "dir"),
+ FILES((short)5, "files"),
+ SET_TIME((short)6, "setTime");
private static final java.util.Map<java.lang.String, _Fields> byName = new java.util.HashMap<java.lang.String, _Fields>();
@@ -24367,13 +24360,11 @@ public static _Fields findByThriftId(int fieldId) {
return CREDENTIALS;
case 3: // TID
return TID;
- case 4: // KEY_EXTENT
- return KEY_EXTENT;
- case 5: // DIR
+ case 4: // DIR
return DIR;
- case 6: // FILES
+ case 5: // FILES
return FILES;
- case 7: // SET_TIME
+ case 6: // SET_TIME
return SET_TIME;
default:
return null;
@@ -24427,14 +24418,14 @@ public short getThriftFieldId() {
new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class)));
tmpMap.put(_Fields.TID, new org.apache.thrift.meta_data.FieldMetaData("tid", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.KEY_EXTENT, new org.apache.thrift.meta_data.FieldMetaData("keyExtent", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class)));
tmpMap.put(_Fields.DIR, new org.apache.thrift.meta_data.FieldMetaData("dir", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.FILES, new org.apache.thrift.meta_data.FieldMetaData("files", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo.class))));
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class),
+ new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo.class)))));
tmpMap.put(_Fields.SET_TIME, new org.apache.thrift.meta_data.FieldMetaData("setTime", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
@@ -24448,9 +24439,8 @@ public loadFiles_args(
org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials,
long tid,
- org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent,
java.lang.String dir,
- java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files,
+ java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files,
boolean setTime)
{
this();
@@ -24458,7 +24448,6 @@ public loadFiles_args(
this.credentials = credentials;
this.tid = tid;
setTidIsSet(true);
- this.keyExtent = keyExtent;
this.dir = dir;
this.files = files;
this.setTime = setTime;
@@ -24477,22 +24466,30 @@ public loadFiles_args(loadFiles_args other) {
this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials);
}
this.tid = other.tid;
- if (other.isSetKeyExtent()) {
- this.keyExtent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other.keyExtent);
- }
if (other.isSetDir()) {
this.dir = other.dir;
}
if (other.isSetFiles()) {
- java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> __this__files = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>(other.files.size());
- for (java.util.Map.Entry<java.lang.String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> other_element : other.files.entrySet()) {
+ java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> __this__files = new java.util.HashMap<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>>(other.files.size());
+ for (java.util.Map.Entry<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> other_element : other.files.entrySet()) {
- java.lang.String other_element_key = other_element.getKey();
- org.apache.accumulo.core.dataImpl.thrift.MapFileInfo other_element_value = other_element.getValue();
+ org.apache.accumulo.core.dataImpl.thrift.TKeyExtent other_element_key = other_element.getKey();
+ java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> other_element_value = other_element.getValue();
- java.lang.String __this__files_copy_key = other_element_key;
+ org.apache.accumulo.core.dataImpl.thrift.TKeyExtent __this__files_copy_key = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other_element_key);
- org.apache.accumulo.core.dataImpl.thrift.MapFileInfo __this__files_copy_value = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo(other_element_value);
+ java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> __this__files_copy_value = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>(other_element_value.size());
+ for (java.util.Map.Entry<java.lang.String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> other_element_value_element : other_element_value.entrySet()) {
+
+ java.lang.String other_element_value_element_key = other_element_value_element.getKey();
+ org.apache.accumulo.core.dataImpl.thrift.MapFileInfo other_element_value_element_value = other_element_value_element.getValue();
+
+ java.lang.String __this__files_copy_value_copy_key = other_element_value_element_key;
+
+ org.apache.accumulo.core.dataImpl.thrift.MapFileInfo __this__files_copy_value_copy_value = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo(other_element_value_element_value);
+
+ __this__files_copy_value.put(__this__files_copy_value_copy_key, __this__files_copy_value_copy_value);
+ }
__this__files.put(__this__files_copy_key, __this__files_copy_value);
}
@@ -24511,7 +24508,6 @@ public void clear() {
this.credentials = null;
setTidIsSet(false);
this.tid = 0;
- this.keyExtent = null;
this.dir = null;
this.files = null;
setSetTimeIsSet(false);
@@ -24589,30 +24585,6 @@ public void setTidIsSet(boolean value) {
__isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __TID_ISSET_ID, value);
}
- public org.apache.accumulo.core.dataImpl.thrift.TKeyExtent getKeyExtent() {
- return this.keyExtent;
- }
-
- public loadFiles_args setKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent keyExtent) {
- this.keyExtent = keyExtent;
- return this;
- }
-
- public void unsetKeyExtent() {
- this.keyExtent = null;
- }
-
- /** Returns true if field keyExtent is set (has been assigned a value) and false otherwise */
- public boolean isSetKeyExtent() {
- return this.keyExtent != null;
- }
-
- public void setKeyExtentIsSet(boolean value) {
- if (!value) {
- this.keyExtent = null;
- }
- }
-
public java.lang.String getDir() {
return this.dir;
}
@@ -24641,18 +24613,18 @@ public int getFilesSize() {
return (this.files == null) ? 0 : this.files.size();
}
- public void putToFiles(java.lang.String key, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo val) {
+ public void putToFiles(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent key, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> val) {
if (this.files == null) {
- this.files = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>();
+ this.files = new java.util.HashMap<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>>();
}
this.files.put(key, val);
}
- public java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> getFiles() {
+ public java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> getFiles() {
return this.files;
}
- public loadFiles_args setFiles(java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> files) {
+ public loadFiles_args setFiles(java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> files) {
this.files = files;
return this;
}
@@ -24721,14 +24693,6 @@ public void setFieldValue(_Fields field, java.lang.Object value) {
}
break;
- case KEY_EXTENT:
- if (value == null) {
- unsetKeyExtent();
- } else {
- setKeyExtent((org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)value);
- }
- break;
-
case DIR:
if (value == null) {
unsetDir();
@@ -24741,7 +24705,7 @@ public void setFieldValue(_Fields field, java.lang.Object value) {
if (value == null) {
unsetFiles();
} else {
- setFiles((java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>)value);
+ setFiles((java.util.Map<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>>)value);
}
break;
@@ -24767,9 +24731,6 @@ public void setFieldValue(_Fields field, java.lang.Object value) {
case TID:
return getTid();
- case KEY_EXTENT:
- return getKeyExtent();
-
case DIR:
return getDir();
@@ -24796,8 +24757,6 @@ public boolean isSet(_Fields field) {
return isSetCredentials();
case TID:
return isSetTid();
- case KEY_EXTENT:
- return isSetKeyExtent();
case DIR:
return isSetDir();
case FILES:
@@ -24850,15 +24809,6 @@ public boolean equals(loadFiles_args that) {
return false;
}
- boolean this_present_keyExtent = true && this.isSetKeyExtent();
- boolean that_present_keyExtent = true && that.isSetKeyExtent();
- if (this_present_keyExtent || that_present_keyExtent) {
- if (!(this_present_keyExtent && that_present_keyExtent))
- return false;
- if (!this.keyExtent.equals(that.keyExtent))
- return false;
- }
-
boolean this_present_dir = true && this.isSetDir();
boolean that_present_dir = true && that.isSetDir();
if (this_present_dir || that_present_dir) {
@@ -24903,10 +24853,6 @@ public int hashCode() {
hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(tid);
- hashCode = hashCode * 8191 + ((isSetKeyExtent()) ? 131071 : 524287);
- if (isSetKeyExtent())
- hashCode = hashCode * 8191 + keyExtent.hashCode();
-
hashCode = hashCode * 8191 + ((isSetDir()) ? 131071 : 524287);
if (isSetDir())
hashCode = hashCode * 8191 + dir.hashCode();
@@ -24958,16 +24904,6 @@ public int compareTo(loadFiles_args other) {
return lastComparison;
}
}
- lastComparison = java.lang.Boolean.valueOf(isSetKeyExtent()).compareTo(other.isSetKeyExtent());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetKeyExtent()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.keyExtent, other.keyExtent);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
lastComparison = java.lang.Boolean.valueOf(isSetDir()).compareTo(other.isSetDir());
if (lastComparison != 0) {
return lastComparison;
@@ -25038,14 +24974,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.
sb.append(this.tid);
first = false;
if (!first) sb.append(", ");
- sb.append("keyExtent:");
- if (this.keyExtent == null) {
- sb.append("null");
- } else {
- sb.append(this.keyExtent);
- }
- first = false;
- if (!first) sb.append(", ");
sb.append("dir:");
if (this.dir == null) {
sb.append("null");
@@ -25078,9 +25006,6 @@ public void validate() throws org.apache.thrift.TException {
if (credentials != null) {
credentials.validate();
}
- if (keyExtent != null) {
- keyExtent.validate();
- }
}
private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
@@ -25145,16 +25070,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, loadFiles_args stru
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
- case 4: // KEY_EXTENT
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.keyExtent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
- struct.keyExtent.read(iprot);
- struct.setKeyExtentIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // DIR
+ case 4: // DIR
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.dir = iprot.readString();
struct.setDirIsSet(true);
@@ -25162,18 +25078,31 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, loadFiles_args stru
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
- case 6: // FILES
+ case 5: // FILES
if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
{
org.apache.thrift.protocol.TMap _map320 = iprot.readMapBegin();
- struct.files = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>(2*_map320.size);
- java.lang.String _key321;
- org.apache.accumulo.core.dataImpl.thrift.MapFileInfo _val322;
+ struct.files = new java.util.HashMap<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>>(2*_map320.size);
+ org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _key321;
+ java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> _val322;
for (int _i323 = 0; _i323 < _map320.size; ++_i323)
{
- _key321 = iprot.readString();
- _val322 = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo();
- _val322.read(iprot);
+ _key321 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
+ _key321.read(iprot);
+ {
+ org.apache.thrift.protocol.TMap _map324 = iprot.readMapBegin();
+ _val322 = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>(2*_map324.size);
+ java.lang.String _key325;
+ org.apache.accumulo.core.dataImpl.thrift.MapFileInfo _val326;
+ for (int _i327 = 0; _i327 < _map324.size; ++_i327)
+ {
+ _key325 = iprot.readString();
+ _val326 = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo();
+ _val326.read(iprot);
+ _val322.put(_key325, _val326);
+ }
+ iprot.readMapEnd();
+ }
struct.files.put(_key321, _val322);
}
iprot.readMapEnd();
@@ -25183,7 +25112,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, loadFiles_args stru
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
- case 7: // SET_TIME
+ case 6: // SET_TIME
if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
struct.setTime = iprot.readBool();
struct.setSetTimeIsSet(true);
@@ -25219,11 +25148,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, loadFiles_args str
oprot.writeFieldBegin(TID_FIELD_DESC);
oprot.writeI64(struct.tid);
oprot.writeFieldEnd();
- if (struct.keyExtent != null) {
- oprot.writeFieldBegin(KEY_EXTENT_FIELD_DESC);
- struct.keyExtent.write(oprot);
- oprot.writeFieldEnd();
- }
if (struct.dir != null) {
oprot.writeFieldBegin(DIR_FIELD_DESC);
oprot.writeString(struct.dir);
@@ -25232,11 +25156,19 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, loadFiles_args str
if (struct.files != null) {
oprot.writeFieldBegin(FILES_FIELD_DESC);
{
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.files.size()));
- for (java.util.Map.Entry<java.lang.String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> _iter324 : struct.files.entrySet())
+ oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.MAP, struct.files.size()));
+ for (java.util.Map.Entry<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> _iter328 : struct.files.entrySet())
{
- oprot.writeString(_iter324.getKey());
- _iter324.getValue().write(oprot);
+ _iter328.getKey().write(oprot);
+ {
+ oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, _iter328.getValue().size()));
+ for (java.util.Map.Entry<java.lang.String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> _iter329 : _iter328.getValue().entrySet())
+ {
+ oprot.writeString(_iter329.getKey());
+ _iter329.getValue().write(oprot);
+ }
+ oprot.writeMapEnd();
+ }
}
oprot.writeMapEnd();
}
@@ -25272,19 +25204,16 @@ public void write(org.apache.thrift.protocol.TProtocol prot, loadFiles_args stru
if (struct.isSetTid()) {
optionals.set(2);
}
- if (struct.isSetKeyExtent()) {
- optionals.set(3);
- }
if (struct.isSetDir()) {
- optionals.set(4);
+ optionals.set(3);
}
if (struct.isSetFiles()) {
- optionals.set(5);
+ optionals.set(4);
}
if (struct.isSetSetTime()) {
- optionals.set(6);
+ optionals.set(5);
}
- oprot.writeBitSet(optionals, 7);
+ oprot.writeBitSet(optionals, 6);
if (struct.isSetTinfo()) {
struct.tinfo.write(oprot);
}
@@ -25294,19 +25223,23 @@ public void write(org.apache.thrift.protocol.TProtocol prot, loadFiles_args stru
if (struct.isSetTid()) {
oprot.writeI64(struct.tid);
}
- if (struct.isSetKeyExtent()) {
- struct.keyExtent.write(oprot);
- }
if (struct.isSetDir()) {
oprot.writeString(struct.dir);
}
if (struct.isSetFiles()) {
{
oprot.writeI32(struct.files.size());
- for (java.util.Map.Entry<java.lang.String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> _iter325 : struct.files.entrySet())
+ for (java.util.Map.Entry<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent, java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>> _iter330 : struct.files.entrySet())
{
- oprot.writeString(_iter325.getKey());
- _iter325.getValue().write(oprot);
+ _iter330.getKey().write(oprot);
+ {
+ oprot.writeI32(_iter330.getValue().size());
+ for (java.util.Map.Entry<java.lang.String, org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> _iter331 : _iter330.getValue().entrySet())
+ {
+ oprot.writeString(_iter331.getKey());
+ _iter331.getValue().write(oprot);
+ }
+ }
}
}
}
@@ -25318,7 +25251,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, loadFiles_args stru
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, loadFiles_args struct) throws org.apache.thrift.TException {
org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot;
- java.util.BitSet incoming = iprot.readBitSet(7);
+ java.util.BitSet incoming = iprot.readBitSet(6);
if (incoming.get(0)) {
struct.tinfo = new org.apache.accumulo.core.trace.thrift.TInfo();
struct.tinfo.read(iprot);
@@ -25334,31 +25267,38 @@ public void read(org.apache.thrift.protocol.TProtocol prot, loadFiles_args struc
struct.setTidIsSet(true);
}
if (incoming.get(3)) {
- struct.keyExtent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
- struct.keyExtent.read(iprot);
- struct.setKeyExtentIsSet(true);
- }
- if (incoming.get(4)) {
struct.dir = iprot.readString();
struct.setDirIsSet(true);
}
- if (incoming.get(5)) {
+ if (incoming.get(4)) {
{
- org.apache.thrift.protocol.TMap _map326 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.files = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>(2*_map326.size);
- java.lang.String _key327;
- org.apache.accumulo.core.dataImpl.thrift.MapFileInfo _val328;
- for (int _i329 = 0; _i329 < _map326.size; ++_i329)
+ org.apache.thrift.protocol.TMap _map332 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRUCT, org.apache.thrift.protocol.TType.MAP, iprot.readI32());
+ struct.files = new java.util.HashMap<org.apache.accumulo.core.dataImpl.thrift.TKeyExtent,java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>>(2*_map332.size);
+ org.apache.accumulo.core.dataImpl.thrift.TKeyExtent _key333;
+ java.util.Map<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo> _val334;
+ for (int _i335 = 0; _i335 < _map332.size; ++_i335)
{
- _key327 = iprot.readString();
- _val328 = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo();
- _val328.read(iprot);
- struct.files.put(_key327, _val328);
+ _key333 = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent();
+ _key333.read(iprot);
+ {
+ org.apache.thrift.protocol.TMap _map336 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ _val334 = new java.util.HashMap<java.lang.String,org.apache.accumulo.core.dataImpl.thrift.MapFileInfo>(2*_map336.size);
+ java.lang.String _key337;
+ org.apache.accumulo.core.dataImpl.thrift.MapFileInfo _val338;
+ for (int _i339 = 0; _i339 < _map336.size; ++_i339)
+ {
+ _key337 = iprot.readString();
+ _val338 = new org.apache.accumulo.core.dataImpl.thrift.MapFileInfo();
+ _val338.read(iprot);
+ _val334.put(_key337, _val338);
+ }
+ }
+ struct.files.put(_key333, _val334);
}
}
struct.setFilesIsSet(true);
}
- if (incoming.get(6)) {
+ if (incoming.get(5)) {
struct.setTime = iprot.readBool();
struct.setSetTimeIsSet(true);
}
@@ -33283,14 +33223,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getTabletStats_resu
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list330 = iprot.readListBegin();
- struct.success = new java.util.ArrayList<TabletStats>(_list330.size);
- TabletStats _elem331;
- for (int _i332 = 0; _i332 < _list330.size; ++_i332)
+ org.apache.thrift.protocol.TList _list340 = iprot.readListBegin();
+ struct.success = new java.util.ArrayList<TabletStats>(_list340.size);
+ TabletStats _elem341;
+ for (int _i342 = 0; _i342 < _list340.size; ++_i342)
{
- _elem331 = new TabletStats();
- _elem331.read(iprot);
- struct.success.add(_elem331);
+ _elem341 = new TabletStats();
+ _elem341.read(iprot);
+ struct.success.add(_elem341);
}
iprot.readListEnd();
}
@@ -33327,9 +33267,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getTabletStats_res
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (TabletStats _iter333 : struct.success)
+ for (TabletStats _iter343 : struct.success)
{
- _iter333.write(oprot);
+ _iter343.write(oprot);
}
oprot.writeListEnd();
}
@@ -33368,9 +33308,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getTabletStats_resu
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (TabletStats _iter334 : struct.success)
+ for (TabletStats _iter344 : struct.success)
{
- _iter334.write(oprot);
+ _iter344.write(oprot);
}
}
}
@@ -33385,14 +33325,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getTabletStats_resul
java.util.BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list335 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new java.util.ArrayList<TabletStats>(_list335.size);
- TabletStats _elem336;
- for (int _i337 = 0; _i337 < _list335.size; ++_i337)
+ org.apache.thrift.protocol.TList _list345 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.success = new java.util.ArrayList<TabletStats>(_list345.size);
+ TabletStats _elem346;
+ for (int _i347 = 0; _i347 < _list345.size; ++_i347)
{
- _elem336 = new TabletStats();
- _elem336.read(iprot);
- struct.success.add(_elem336);
+ _elem346 = new TabletStats();
+ _elem346.read(iprot);
+ struct.success.add(_elem346);
}
}
struct.setSuccessIsSet(true);
@@ -36754,14 +36694,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveScans_resu
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list338 = iprot.readListBegin();
- struct.success = new java.util.ArrayList<ActiveScan>(_list338.size);
- ActiveScan _elem339;
- for (int _i340 = 0; _i340 < _list338.size; ++_i340)
+ org.apache.thrift.protocol.TList _list348 = iprot.readListBegin();
+ struct.success = new java.util.ArrayList<ActiveScan>(_list348.size);
+ ActiveScan _elem349;
+ for (int _i350 = 0; _i350 < _list348.size; ++_i350)
{
- _elem339 = new ActiveScan();
- _elem339.read(iprot);
- struct.success.add(_elem339);
+ _elem349 = new ActiveScan();
+ _elem349.read(iprot);
+ struct.success.add(_elem349);
}
iprot.readListEnd();
}
@@ -36798,9 +36738,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveScans_res
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (ActiveScan _iter341 : struct.success)
+ for (ActiveScan _iter351 : struct.success)
{
- _iter341.write(oprot);
+ _iter351.write(oprot);
}
oprot.writeListEnd();
}
@@ -36839,9 +36779,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveScans_resu
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (ActiveScan _iter342 : struct.success)
+ for (ActiveScan _iter352 : struct.success)
{
- _iter342.write(oprot);
+ _iter352.write(oprot);
}
}
}
@@ -36856,14 +36796,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveScans_resul
java.util.BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list343 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new java.util.ArrayList<ActiveScan>(_list343.size);
- ActiveScan _elem344;
- for (int _i345 = 0; _i345 < _list343.size; ++_i345)
+ org.apache.thrift.protocol.TList _list353 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.success = new java.util.ArrayList<ActiveScan>(_list353.size);
+ ActiveScan _elem354;
+ for (int _i355 = 0; _i355 < _list353.size; ++_i355)
{
- _elem344 = new ActiveScan();
- _elem344.read(iprot);
- struct.success.add(_elem344);
+ _elem354 = new ActiveScan();
+ _elem354.read(iprot);
+ struct.success.add(_elem354);
}
}
struct.setSuccessIsSet(true);
@@ -37752,14 +37692,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveCompaction
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list346 = iprot.readListBegin();
- struct.success = new java.util.ArrayList<ActiveCompaction>(_list346.size);
- ActiveCompaction _elem347;
- for (int _i348 = 0; _i348 < _list346.size; ++_i348)
+ org.apache.thrift.protocol.TList _list356 = iprot.readListBegin();
+ struct.success = new java.util.ArrayList<ActiveCompaction>(_list356.size);
+ ActiveCompaction _elem357;
+ for (int _i358 = 0; _i358 < _list356.size; ++_i358)
{
- _elem347 = new ActiveCompaction();
- _elem347.read(iprot);
- struct.success.add(_elem347);
+ _elem357 = new ActiveCompaction();
+ _elem357.read(iprot);
+ struct.success.add(_elem357);
}
iprot.readListEnd();
}
@@ -37796,9 +37736,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveCompactio
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size()));
- for (ActiveCompaction _iter349 : struct.success)
+ for (ActiveCompaction _iter359 : struct.success)
{
- _iter349.write(oprot);
+ _iter359.write(oprot);
}
oprot.writeListEnd();
}
@@ -37837,9 +37777,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveCompaction
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (ActiveCompaction _iter350 : struct.success)
+ for (ActiveCompaction _iter360 : struct.success)
{
- _iter350.write(oprot);
+ _iter360.write(oprot);
}
}
}
@@ -37854,14 +37794,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveCompactions
java.util.BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list351 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.success = new java.util.ArrayList<ActiveCompaction>(_list351.size);
- ActiveCompaction _elem352;
- for (int _i353 = 0; _i353 < _list351.size; ++_i353)
+ org.apache.thrift.protocol.TList _list361 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.success = new java.util.ArrayList<ActiveCompaction>(_list361.size);
+ ActiveCompaction _elem362;
+ for (int _i363 = 0; _i363 < _list361.size; ++_i363)
{
- _elem352 = new ActiveCompaction();
- _elem352.read(iprot);
- struct.success.add(_elem352);
+ _elem362 = new ActiveCompaction();
+ _elem362.read(iprot);
+ struct.success.add(_elem362);
}
}
struct.setSuccessIsSet(true);
@@ -38376,13 +38316,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, removeLogs_args str
case 3: // FILENAMES
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list354 = iprot.readListBegin();
- struct.filenames = new java.util.ArrayList<java.lang.String>(_list354.size);
- java.lang.String _elem355;
- for (int _i356 = 0; _i356 < _list354.size; ++_i356)
+ org.apache.thrift.protocol.TList _list364 = iprot.readListBegin();
+ struct.filenames = new java.util.ArrayList<java.lang.String>(_list364.size);
+ java.lang.String _elem365;
+ for (int _i366 = 0; _i366 < _list364.size; ++_i366)
{
- _elem355 = iprot.readString();
- struct.filenames.add(_elem355);
+ _elem365 = iprot.readString();
+ struct.filenames.add(_elem365);
}
iprot.readListEnd();
}
@@ -38420,9 +38360,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, removeLogs_args st
oprot.writeFieldBegin(FILENAMES_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.filenames.size()));
- for (java.lang.String _iter357 : struct.filenames)
+ for (java.lang.String _iter367 : struct.filenames)
{
- oprot.writeString(_iter357);
+ oprot.writeString(_iter367);
}
oprot.writeListEnd();
}
@@ -38465,9 +38405,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, removeLogs_args str
if (struct.isSetFilenames()) {
{
oprot.writeI32(struct.filenames.size());
- for (java.lang.String _iter358 : struct.filenames)
+ for (java.lang.String _iter368 : struct.filenames)
{
- oprot.writeString(_iter358);
+ oprot.writeString(_iter368);
}
}
}
@@ -38489,13 +38429,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, removeLogs_args stru
}
if (incoming.get(2)) {
{
- org.apache.thrift.protocol.TList _list359 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.filenames = new java.util.ArrayList<java.lang.String>(_list359.size);
- java.lang.String _elem360;
- for (int _i361 = 0; _i361 < _list359.size; ++_i361)
+ org.apache.thrift.protocol.TList _list369 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ struct.filenames = new java.util.ArrayList<java.lang.String>(_list369.size);
+ java.lang.String _elem370;
+ for (int _i371 = 0; _i371 < _list369.size; ++_i371)
{
- _elem360 = iprot.readString();
- struct.filenames.add(_elem360);
+ _elem370 = iprot.readString();
+ struct.filenames.add(_elem370);
}
}
struct.setFilenamesIsSet(true);
@@ -39295,13 +39235,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveLogs_resul
case 0: // SUCCESS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list362 = iprot.readListBegin();
- struct.success = new java.util.ArrayList<java.lang.String>(_list362.size);
- java.lang.String _elem363;
- for (int _i364 = 0; _i364 < _list362.size; ++_i364)
+ org.apache.thrift.protocol.TList _list372 = iprot.readListBegin();
+ struct.success = new java.util.ArrayList<java.lang.String>(_list372.size);
+ java.lang.String _elem373;
+ for (int _i374 = 0; _i374 < _list372.size; ++_i374)
{
- _elem363 = iprot.readString();
- struct.success.add(_elem363);
+ _elem373 = iprot.readString();
+ struct.success.add(_elem373);
}
iprot.readListEnd();
}
@@ -39329,9 +39269,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveLogs_resu
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.success.size()));
- for (java.lang.String _iter365 : struct.success)
+ for (java.lang.String _iter375 : struct.success)
{
- oprot.writeString(_iter365);
+ oprot.writeString(_iter375);
}
oprot.writeListEnd();
}
@@ -39362,9 +39302,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveLogs_resul
if (struct.isSetSuccess()) {
{
oprot.writeI32(struct.success.size());
- for (java.lang.String _iter366 : struct.success)
+ for (java.lang.String _iter376 : struct.success)
{
- oprot.writeString(_iter366);
+ oprot.writeString(_iter376);
}
}
}
@@ -39376,13 +39316,13 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveLogs_result
java.util.BitSet incoming = iprot.readBitSet(1);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list367 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.success = new java.util.ArrayList<java.lang.String>(_list367.size);
- java.lang.String _elem368;
- for (int _i369 = 0; _i369 < _list367.size; ++_i369)
+ org.apache.thrift.protocol.TList _list377 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ struct.success = new java.util.ArrayList<java.lang.String>(_list377.size);
+ java.lang.String _elem378;
+ for (int _i379 = 0; _i379 < _list377.size; ++_i379)
{
- _elem368 = iprot.readString();
- struct.success.add(_elem368);
+ _elem378 = iprot.readString();
+ struct.success.add(_elem378);
}
}
struct.setSuccessIsSet(true);
@@ -42415,26 +42355,26 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, startGetSummariesFr
case 4: // FILES
if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
{
- org.apache.thrift.protocol.TMap _map370 = iprot.readMapBegin();
- struct.files = new java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map370.size);
- java.lang.String _key371;
- java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val372;
- for (int _i373 = 0; _i373 < _map370.size; ++_i373)
+ org.apache.thrift.protocol.TMap _map380 = iprot.readMapBegin();
+ struct.files = new java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map380.size);
+ java.lang.String _key381;
+ java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val382;
+ for (int _i383 = 0; _i383 < _map380.size; ++_i383)
{
- _key371 = iprot.readString();
+ _key381 = iprot.readString();
{
- org.apache.thrift.protocol.TList _list374 = iprot.readListBegin();
- _val372 = new java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list374.size);
- org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem375;
- for (int _i376 = 0; _i376 < _list374.size; ++_i376)
+ org.apache.thrift.protocol.TList _list384 = iprot.readListBegin();
+ _val382 = new java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list384.size);
+ org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem385;
+ for (int _i386 = 0; _i386 < _list384.size; ++_i386)
{
- _elem375 = new org.apache.accumulo.core.dataImpl.thrift.TRowRange();
- _elem375.read(iprot);
- _val372.add(_elem375);
+ _elem385 = new org.apache.accumulo.core.dataImpl.thrift.TRowRange();
+ _elem385.read(iprot);
+ _val382.add(_elem385);
}
iprot.readListEnd();
}
- struct.files.put(_key371, _val372);
+ struct.files.put(_key381, _val382);
}
iprot.readMapEnd();
}
@@ -42477,14 +42417,14 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, startGetSummariesF
oprot.writeFieldBegin(FILES_FIELD_DESC);
{
oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, struct.files.size()));
- for (java.util.Map.Entry<java.lang.String, java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter377 : struct.files.entrySet())
+ for (java.util.Map.Entry<java.lang.String, java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter387 : struct.files.entrySet())
{
- oprot.writeString(_iter377.getKey());
+ oprot.writeString(_iter387.getKey());
{
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter377.getValue().size()));
- for (org.apache.accumulo.core.dataImpl.thrift.TRowRange _iter378 : _iter377.getValue())
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, _iter387.getValue().size()));
+ for (org.apache.accumulo.core.dataImpl.thrift.TRowRange _iter388 : _iter387.getValue())
{
- _iter378.write(oprot);
+ _iter388.write(oprot);
}
oprot.writeListEnd();
}
@@ -42536,14 +42476,14 @@ public void write(org.apache.thrift.protocol.TProtocol prot, startGetSummariesFr
if (struct.isSetFiles()) {
{
oprot.writeI32(struct.files.size());
- for (java.util.Map.Entry<java.lang.String, java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter379 : struct.files.entrySet())
+ for (java.util.Map.Entry<java.lang.String, java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>> _iter389 : struct.files.entrySet())
{
- oprot.writeString(_iter379.getKey());
+ oprot.writeString(_iter389.getKey());
{
- oprot.writeI32(_iter379.getValue().size());
- for (org.apache.accumulo.core.dataImpl.thrift.TRowRange _iter380 : _iter379.getValue())
+ oprot.writeI32(_iter389.getValue().size());
+ for (org.apache.accumulo.core.dataImpl.thrift.TRowRange _iter390 : _iter389.getValue())
{
- _iter380.write(oprot);
+ _iter390.write(oprot);
}
}
}
@@ -42572,25 +42512,25 @@ public void read(org.apache.thrift.protocol.TProtocol prot, startGetSummariesFro
}
if (incoming.get(3)) {
{
- org.apache.thrift.protocol.TMap _map381 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, iprot.readI32());
- struct.files = new java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map381.size);
- java.lang.String _key382;
- java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val383;
- for (int _i384 = 0; _i384 < _map381.size; ++_i384)
+ org.apache.thrift.protocol.TMap _map391 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.LIST, iprot.readI32());
+ struct.files = new java.util.HashMap<java.lang.String,java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange>>(2*_map391.size);
+ java.lang.String _key392;
+ java.util.List<org.apache.accumulo.core.dataImpl.thrift.TRowRange> _val393;
+ for (int _i394 = 0; _i394 < _map391.size; ++_i394)
{
- _key382 = iprot.readString();
+ _key392 = iprot.readString();
{
- org.apache.thrift.protocol.TList _list385 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- _val383 = new java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list385.size);
- org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem386;
- for (int _i387 = 0; _i387 < _list385.size; ++_i387)
+ org.apache.thrift.protocol.TList _list395 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ _val393 = new java.util.ArrayList<org.apache.accumulo.core.dataImpl.thrift.TRowRange>(_list395.size);
+ org.apache.accumulo.core.dataImpl.thrift.TRowRange _elem396;
+ for (int _i397 = 0; _i397 < _list395.size; ++_i397)
{
- _elem386 = new org.apache.accumulo.core.dataImpl.thrift.TRowRange();
- _elem386.read(iprot);
- _val383.add(_elem386);
+ _elem396 = new org.apache.accumulo.core.dataImpl.thrift.TRowRange();
+ _elem396.read(iprot);
+ _val393.add(_elem396);
}
}
- struct.files.put(_key382, _val383);
+ struct.files.put(_key392, _val393);
}
}
struct.setFilesIsSet(true);
diff --git a/core/src/main/thrift/tabletserver.thrift b/core/src/main/thrift/tabletserver.thrift
index 44e69ba0a5..8439a82625 100644
--- a/core/src/main/thrift/tabletserver.thrift
+++ b/core/src/main/thrift/tabletserver.thrift
@@ -213,7 +213,7 @@ service TabletClientService extends client.ClientService {
// on success, returns an empty list
list<data.TKeyExtent> bulkImport(3:trace.TInfo tinfo, 1:security.TCredentials credentials, 4:i64 tid, 2:data.TabletFiles files, 5:bool setTime) throws (1:client.ThriftSecurityException sec),
- oneway void loadFiles(1:trace.TInfo tinfo, 2:security.TCredentials credentials, 3:i64 tid, 4:data.TKeyExtent keyExtent, 5:string dir, 6:map<string, data.MapFileInfo> files, 7:bool setTime);
+ oneway void loadFiles(1:trace.TInfo tinfo, 2:security.TCredentials credentials, 3:i64 tid, 4:string dir, 5:map<data.TKeyExtent, map<string, data.MapFileInfo>> files, 6:bool setTime);
void splitTablet(4:trace.TInfo tinfo, 1:security.TCredentials credentials, 2:data.TKeyExtent extent, 3:binary splitPoint) throws (1:client.ThriftSecurityException sec, 2:NotServingTabletException nste)
diff --git a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java
index b269bd58eb..88e3ad17cd 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/TransactionWatcher.java
@@ -122,28 +122,41 @@ public boolean transactionComplete(String type, long tid) throws Exception {
}
}
+ /**
+ * Run task only if transaction is still active in zookeeper. If the tx is no longer active then
+ * that task is not run and a debug message is logged indicating the task was ignored.
+ */
+ public void runQuietly(String ztxBulk, long tid, Runnable task) {
+ synchronized (counts) {
+ try {
+ if (!arbitrator.transactionAlive(ztxBulk, tid)) {
+ log.debug("Transaction " + tid + " of type " + ztxBulk + " is no longer active.");
+ return;
+ }
+ } catch (Exception e) {
+ log.warn("Unable to check if transaction " + tid + " of type " + ztxBulk + " is alive ", e);
+ return;
+ }
+ increment(tid);
+ }
+ try {
+ task.run();
+ } finally {
+ decrement(tid);
+ }
+ }
+
public <T> T run(String ztxBulk, long tid, Callable<T> callable) throws Exception {
synchronized (counts) {
if (!arbitrator.transactionAlive(ztxBulk, tid)) {
throw new Exception("Transaction " + tid + " of type " + ztxBulk + " is no longer active");
}
- AtomicInteger count = counts.get(tid);
- if (count == null)
- counts.put(tid, count = new AtomicInteger());
- count.incrementAndGet();
+ increment(tid);
}
try {
return callable.call();
} finally {
- synchronized (counts) {
- AtomicInteger count = counts.get(tid);
- if (count == null) {
- log.error("unexpected missing count for transaction {}", tid);
- } else {
- if (count.decrementAndGet() == 0)
- counts.remove(tid);
- }
- }
+ decrement(tid);
}
}
@@ -155,4 +168,22 @@ public boolean isActive(long tid) {
}
}
+ private void increment(long tid) {
+ AtomicInteger count = counts.get(tid);
+ if (count == null)
+ counts.put(tid, count = new AtomicInteger());
+ count.incrementAndGet();
+ }
+
+ private void decrement(long tid) {
+ synchronized (counts) {
+ AtomicInteger count = counts.get(tid);
+ if (count == null) {
+ log.error("unexpected missing count for transaction {}", tid);
+ } else {
+ if (count.decrementAndGet() == 0)
+ counts.remove(tid);
+ }
+ }
+ }
}
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
index b65b3892fa..ce2b4991cb 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
@@ -38,6 +38,7 @@
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.MapFileInfo;
+import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -128,6 +129,12 @@ void start(Path bulkDir, Master master, long tid, boolean setTime) throws Except
// track how many tablets were sent load messages per tablet server
MapCounter<HostAndPort> loadMsgs;
+ // Each RPC to a tablet server needs to check in zookeeper to see if the transaction is still
+ // active. The purpose of this map is to group load request by tablet servers inorder to do less
+ // RPCs. Less RPCs will result in less calls to Zookeeper.
+ Map<HostAndPort,Map<TKeyExtent,Map<String,MapFileInfo>>> loadQueue;
+ private int queuedDataSize = 0;
+
@Override
void start(Path bulkDir, Master master, long tid, boolean setTime) throws Exception {
super.start(bulkDir, master, tid, setTime);
@@ -136,6 +143,52 @@ void start(Path bulkDir, Master master, long tid, boolean setTime) throws Except
fmtTid = String.format("%016x", tid);
loadMsgs = new MapCounter<>();
+
+ loadQueue = new HashMap<>();
+ }
+
+ private void sendQueued(int threshhold) {
+ if (queuedDataSize > threshhold || threshhold == 0) {
+ loadQueue.forEach((server, tabletFiles) -> {
+
+ if (log.isTraceEnabled()) {
+ log.trace("tid {} asking {} to bulk import {} files for {} tablets", fmtTid, server,
+ tabletFiles.values().stream().mapToInt(Map::size).sum(), tabletFiles.size());
+ }
+
+ TabletClientService.Client client = null;
+ try {
+ client = ThriftUtil.getTServerClient(server, master.getContext(), timeInMillis);
+ client.loadFiles(Tracer.traceInfo(), master.getContext().rpcCreds(), tid,
+ bulkDir.toString(), tabletFiles, setTime);
+ } catch (TException ex) {
+ log.debug("rpc failed server: " + server + ", tid:" + fmtTid + " " + ex.getMessage(),
+ ex);
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ });
+
+ loadQueue.clear();
+ queuedDataSize = 0;
+ }
+ }
+
+ private void addToQueue(HostAndPort server, KeyExtent extent,
+ Map<String,MapFileInfo> thriftImports) {
+ if (!thriftImports.isEmpty()) {
+ loadMsgs.increment(server, 1);
+
+ Map<String,MapFileInfo> prev = loadQueue.computeIfAbsent(server, k -> new HashMap<>())
+ .putIfAbsent(extent.toThrift(), thriftImports);
+
+ Preconditions.checkState(prev == null, "Unexpectedly saw extent %s twice", extent);
+
+ // keep a very rough estimate of how much is memory so we can send if over a few megs is
+ // buffered
+ queuedDataSize += thriftImports.keySet().stream().mapToInt(String::length).sum()
+ + server.getHost().length() + 4 + thriftImports.size() * 32;
+ }
}
@Override
@@ -165,30 +218,17 @@ void load(List<TabletMetadata> tablets, Files files) {
}
}
- if (thriftImports.size() > 0) {
- // must always increment this even if there is a comms failure, because it indicates there
- // is work to do
- loadMsgs.increment(server, 1);
- log.trace("tid {} asking {} to bulk import {} files", fmtTid, server,
- thriftImports.size());
- TabletClientService.Client client = null;
- try {
- client = ThriftUtil.getTServerClient(server, master.getContext(), timeInMillis);
- client.loadFiles(Tracer.traceInfo(), master.getContext().rpcCreds(), tid,
- tablet.getExtent().toThrift(), bulkDir.toString(), thriftImports, setTime);
- } catch (TException ex) {
- log.debug("rpc failed server: " + server + ", tid:" + fmtTid + " " + ex.getMessage(),
- ex);
- } finally {
- ThriftUtil.returnClient(client);
- }
- }
+ addToQueue(server, tablet.getExtent(), thriftImports);
}
+ sendQueued(4 * 1024 * 1024);
}
@Override
long finish() {
+
+ sendQueued(0);
+
long sleepTime = 0;
if (loadMsgs.size() > 0) {
// find which tablet server had the most load messages sent to it and sleep 13ms for each
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index db697054c7..0c5aeccb28 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -49,7 +49,6 @@
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -496,39 +495,35 @@ private static long jitter(long ms) {
SecurityErrorCode.PERMISSION_DENIED);
try {
- return watcher.run(Constants.BULK_ARBITRATOR_TYPE, tid, new Callable<List<TKeyExtent>>() {
-
- @Override
- public List<TKeyExtent> call() throws Exception {
- List<TKeyExtent> failures = new ArrayList<>();
-
- for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
- TKeyExtent tke = entry.getKey();
- Map<String,MapFileInfo> fileMap = entry.getValue();
- Map<FileRef,MapFileInfo> fileRefMap = new HashMap<>();
- for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
- Path path = new Path(mapping.getKey());
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
- path = ns.makeQualified(path);
- fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
- }
+ return watcher.run(Constants.BULK_ARBITRATOR_TYPE, tid, () -> {
+ List<TKeyExtent> failures = new ArrayList<>();
+
+ for (Entry<TKeyExtent,Map<String,MapFileInfo>> entry : files.entrySet()) {
+ TKeyExtent tke = entry.getKey();
+ Map<String,MapFileInfo> fileMap = entry.getValue();
+ Map<FileRef,MapFileInfo> fileRefMap = new HashMap<>();
+ for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
+ Path path = new Path(mapping.getKey());
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ path = ns.makeQualified(path);
+ fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
+ }
- Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
- if (importTablet == null) {
+ if (importTablet == null) {
+ failures.add(tke);
+ } else {
+ try {
+ importTablet.importMapFiles(tid, fileRefMap, setTime);
+ } catch (IOException ioe) {
+ log.info("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke),
+ ioe.getMessage());
failures.add(tke);
- } else {
- try {
- importTablet.importMapFiles(tid, fileRefMap, setTime);
- } catch (IOException ioe) {
- log.info("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke),
- ioe.getMessage());
- failures.add(tke);
- }
}
}
- return failures;
}
+ return failures;
});
} catch (RuntimeException e) {
throw e;
@@ -538,31 +533,36 @@ private static long jitter(long ms) {
}
@Override
- public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, TKeyExtent tke,
- String dir, Map<String,MapFileInfo> fileMap, boolean setTime)
+ public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, String dir,
+ Map<TKeyExtent,Map<String,MapFileInfo>> tabletImports, boolean setTime)
throws ThriftSecurityException {
if (!security.canPerformSystemActions(credentials))
throw new ThriftSecurityException(credentials.getPrincipal(),
SecurityErrorCode.PERMISSION_DENIED);
- Map<FileRef,MapFileInfo> fileRefMap = new HashMap<>();
- for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
- Path path = new Path(dir, mapping.getKey());
- FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
- path = ns.makeQualified(path);
- fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
- }
+ watcher.runQuietly(Constants.BULK_ARBITRATOR_TYPE, tid, () -> {
+ tabletImports.forEach((tke, fileMap) -> {
+ Map<FileRef,MapFileInfo> fileRefMap = new HashMap<>();
+ for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
+ Path path = new Path(dir, mapping.getKey());
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
+ path = ns.makeQualified(path);
+ fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
+ }
- Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+ Tablet importTablet = onlineTablets.get(new KeyExtent(tke));
+
+ if (importTablet != null) {
+ try {
+ importTablet.importMapFiles(tid, fileRefMap, setTime);
+ } catch (IOException ioe) {
+ log.debug("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke),
+ ioe.getMessage());
+ }
+ }
+ });
+ });
- if (importTablet != null) {
- try {
- importTablet.importMapFiles(tid, fileRefMap, setTime);
- } catch (IOException ioe) {
- log.info("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke),
- ioe.getMessage());
- }
- }
}
private ScanDispatcher getScanDispatcher(KeyExtent extent) {
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkFailureIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkFailureIT.java
index b6c5eb059a..e364892eba 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkFailureIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkFailureIT.java
@@ -20,9 +20,10 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -31,7 +32,11 @@
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.rfile.RFile;
@@ -46,12 +51,12 @@
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.MapFileInfo;
-import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
import org.apache.accumulo.core.trace.Tracer;
import org.apache.accumulo.core.util.HostAndPort;
@@ -66,6 +71,8 @@
import org.apache.hadoop.io.Text;
import org.apache.thrift.TApplicationException;
import org.apache.thrift.TServiceClient;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException;
import org.junit.Test;
import com.google.common.collect.ImmutableMap;
@@ -73,6 +80,22 @@
public class BulkFailureIT extends AccumuloClusterHarness {
+ static interface Loader {
+ void load(long txid, ClientContext context, KeyExtent extent, Path path, long size,
+ boolean expectFailure) throws Exception;
+ }
+
+ @Test
+ public void testImportCompactionImport() throws Exception {
+ String tables[] = getUniqueNames(2);
+
+ // run test calling old bulk import RPCs
+ runTest(tables[0], 99999999L, BulkFailureIT::oldLoad);
+
+ // run test calling new bulk import RPCs
+ runTest(tables[1], 22222222L, BulkFailureIT::newLoad);
+ }
+
/**
* This test verifies two things. First it ensures that after a bulk imported file is compacted
* that import request are ignored. Second it ensures that after the bulk import transaction is
@@ -80,15 +103,15 @@
* test. Internal (non public API) RPCs and Zookeeper state is manipulated directly. This is the
* only way to interleave compactions with multiple, duplicate import RPC request.
*/
- @Test
- public void testImportCompactionImport() throws Exception {
+ protected void runTest(String table, long fateTxid, Loader loader) throws IOException,
+ AccumuloException, AccumuloSecurityException, TableExistsException, KeeperException,
+ InterruptedException, Exception, FileNotFoundException, TableNotFoundException {
try (AccumuloClient c = createAccumuloClient()) {
- String table = getUniqueNames(1)[0];
SortedMap<Key,Value> testData = createTestData();
FileSystem fs = getCluster().getFileSystem();
- String testFile = createTestFile(testData, fs);
+ String testFile = createTestFile(fateTxid, testData, fs);
c.tableOperations().create(table);
String tableId = c.tableOperations().tableIdMap().get(table);
@@ -96,8 +119,6 @@ public void testImportCompactionImport() throws Exception {
// Table has no splits, so this extent corresponds to the tables single tablet
KeyExtent extent = new KeyExtent(Table.ID.of(tableId), null, null);
- long fateTxid = 99999999L;
-
ServerContext asCtx = getServerContext();
ZooArbitrator.start(asCtx, Constants.BULK_ARBITRATOR_TYPE, fateTxid);
@@ -111,7 +132,7 @@ public void testImportCompactionImport() throws Exception {
Path bulkLoadPath = fs.makeQualified(status.getPath());
// Directly ask the tablet to load the file.
- assignMapFiles(fateTxid, asCtx, extent, bulkLoadPath.toString(), status.getLen());
+ loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), false);
assertEquals(ImmutableSet.of(bulkLoadPath), getFiles(c, extent));
assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent));
@@ -127,7 +148,7 @@ public void testImportCompactionImport() throws Exception {
assertEquals(testData, readTable(table, c));
// this request should be ignored by the tablet
- assignMapFiles(fateTxid, asCtx, extent, bulkLoadPath.toString(), status.getLen());
+ loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), false);
assertEquals(tabletFiles, getFiles(c, extent));
assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent));
@@ -139,7 +160,7 @@ public void testImportCompactionImport() throws Exception {
c.tableOperations().online(table, true);
// this request should be ignored by the tablet
- assignMapFiles(fateTxid, asCtx, extent, bulkLoadPath.toString(), status.getLen());
+ loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), false);
assertEquals(tabletFiles, getFiles(c, extent));
assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent));
@@ -148,16 +169,18 @@ public void testImportCompactionImport() throws Exception {
// After this, all load request should fail.
ZooArbitrator.stop(asCtx, Constants.BULK_ARBITRATOR_TYPE, fateTxid);
- try {
- // expect this to fail
- assignMapFiles(fateTxid, asCtx, extent, bulkLoadPath.toString(), status.getLen());
- fail();
- } catch (TApplicationException tae) {
+ c.securityOperations().grantTablePermission(c.whoami(), MetadataTable.NAME,
+ TablePermission.WRITE);
- }
+ BatchDeleter bd = c.createBatchDeleter(MetadataTable.NAME, Authorizations.EMPTY, 1);
+ bd.setRanges(Collections.singleton(extent.toMetadataRange()));
+ bd.fetchColumnFamily(BulkFileColumnFamily.NAME);
+ bd.delete();
+
+ loader.load(fateTxid, asCtx, extent, bulkLoadPath, status.getLen(), true);
assertEquals(tabletFiles, getFiles(c, extent));
- assertEquals(ImmutableSet.of(bulkLoadPath), getLoaded(c, extent));
+ assertEquals(ImmutableSet.of(), getLoaded(c, extent));
assertEquals(testData, readTable(table, c));
}
}
@@ -171,8 +194,9 @@ public void testImportCompactionImport() throws Exception {
return testData;
}
- private String createTestFile(SortedMap<Key,Value> testData, FileSystem fs) throws IOException {
- Path base = new Path(getCluster().getTemporaryPath(), "testBulk_ICI");
+ private String createTestFile(long txid, SortedMap<Key,Value> testData, FileSystem fs)
+ throws IOException {
+ Path base = new Path(getCluster().getTemporaryPath(), "testBulk_ICI_" + txid);
fs.delete(base, true);
fs.mkdirs(base);
@@ -200,17 +224,17 @@ private String createTestFile(SortedMap<Key,Value> testData, FileSystem fs) thro
return actual;
}
- public Set<Path> getLoaded(AccumuloClient connector, KeyExtent extent)
+ public static Set<Path> getLoaded(AccumuloClient connector, KeyExtent extent)
throws TableNotFoundException {
return getPaths(connector, extent, BulkFileColumnFamily.NAME);
}
- public Set<Path> getFiles(AccumuloClient connector, KeyExtent extent)
+ public static Set<Path> getFiles(AccumuloClient connector, KeyExtent extent)
throws TableNotFoundException {
return getPaths(connector, extent, DataFileColumnFamily.NAME);
}
- private Set<Path> getPaths(AccumuloClient connector, KeyExtent extent, Text fam)
+ private static Set<Path> getPaths(AccumuloClient connector, KeyExtent extent, Text fam)
throws TableNotFoundException {
HashSet<Path> files = new HashSet<>();
@@ -225,30 +249,63 @@ private String createTestFile(SortedMap<Key,Value> testData, FileSystem fs) thro
return files;
}
- private List<KeyExtent> assignMapFiles(long txid, ClientContext context, KeyExtent extent,
- String path, long size) throws Exception {
+ private static void oldLoad(long txid, ClientContext context, KeyExtent extent, Path path,
+ long size, boolean expectFailure) throws Exception {
- TabletLocator locator = TabletLocator.getLocator(context, extent.getTableId());
+ TabletClientService.Iface client = getClient(context, extent);
+ try {
- locator.invalidateCache(extent);
+ Map<String,MapFileInfo> val = ImmutableMap.of(path.toString(), new MapFileInfo(size));
+ Map<KeyExtent,Map<String,MapFileInfo>> files = ImmutableMap.of(extent, val);
- HostAndPort location = HostAndPort
- .fromString(locator.locateTablet(context, new Text(""), false, true).tablet_location);
+ client.bulkImport(Tracer.traceInfo(), context.rpcCreds(), txid,
+ Translator.translate(files, Translators.KET), false);
+ if (expectFailure) {
+ fail("Expected RPC to fail");
+ }
+ } catch (TApplicationException tae) {
+ if (!expectFailure)
+ throw tae;
+ } finally {
+ ThriftUtil.returnClient((TServiceClient) client);
+ }
+ }
- long timeInMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT);
- TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context, timeInMillis);
+ private static void newLoad(long txid, ClientContext context, KeyExtent extent, Path path,
+ long size, boolean expectFailure) throws Exception {
+
+ TabletClientService.Iface client = getClient(context, extent);
try {
- Map<String,MapFileInfo> val = ImmutableMap.of(path, new MapFileInfo(size));
+ Map<String,MapFileInfo> val = ImmutableMap.of(path.getName(), new MapFileInfo(size));
Map<KeyExtent,Map<String,MapFileInfo>> files = ImmutableMap.of(extent, val);
- List<TKeyExtent> failures = client.bulkImport(Tracer.traceInfo(), context.rpcCreds(), txid,
+ client.loadFiles(Tracer.traceInfo(), context.rpcCreds(), txid, path.getParent().toString(),
Translator.translate(files, Translators.KET), false);
- return Translator.translate(failures, Translators.TKET);
+ if (!expectFailure) {
+ while (!getLoaded(context, extent).contains(path)) {
+ Thread.sleep(100);
+ }
+ }
+
} finally {
ThriftUtil.returnClient((TServiceClient) client);
}
+ }
+
+ protected static TabletClientService.Iface getClient(ClientContext context, KeyExtent extent)
+ throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
+ TTransportException {
+ TabletLocator locator = TabletLocator.getLocator(context, extent.getTableId());
+
+ locator.invalidateCache(extent);
+ HostAndPort location = HostAndPort
+ .fromString(locator.locateTablet(context, new Text(""), false, true).tablet_location);
+
+ long timeInMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT);
+ TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context, timeInMillis);
+ return client;
}
}
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
index 4a97105833..8b81b60f1e 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java
@@ -114,8 +114,8 @@ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) {
}
@Override
- public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, TKeyExtent tke,
- String dir, Map<String,MapFileInfo> fileMap, boolean setTime) {}
+ public void loadFiles(TInfo tinfo, TCredentials credentials, long tid, String dir,
+ Map<TKeyExtent,Map<String,MapFileInfo>> fileMap, boolean setTime) {}
@Override
public void closeMultiScan(TInfo tinfo, long scanID) {}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services