You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/07/17 14:58:24 UTC
[10/10] git commit: ACCUMULO-1000 compressed iterator config in
conditional writer RPC calls
ACCUMULO-1000 compressed iterator config in conditional writer RPC calls
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/be54c959
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/be54c959
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/be54c959
Branch: refs/heads/ACCUMULO-1000
Commit: be54c959d2e04556401d62acceaae6a2ddbc62bc
Parents: 3a2fca3
Author: Keith Turner <kt...@apache.org>
Authored: Tue Jul 16 19:17:44 2013 -0400
Committer: Keith Turner <kt...@apache.org>
Committed: Tue Jul 16 19:17:44 2013 -0400
----------------------------------------------------------------------
.../core/client/impl/CompressedIterators.java | 131 +++++
.../core/client/impl/ConditionalWriterImpl.java | 37 +-
.../accumulo/core/data/ArrayByteSequence.java | 13 +
.../org/apache/accumulo/core/data/Mutation.java | 130 +----
.../accumulo/core/data/thrift/TCondition.java | 379 ++-----------
.../accumulo/core/file/rfile/RelativeKey.java | 33 +-
.../thrift/TabletClientService.java | 556 ++++++++++++-------
.../core/util/UnsynchronizedBuffer.java | 195 +++++++
core/src/main/thrift/data.thrift | 3 +-
core/src/main/thrift/tabletserver.thrift | 2 +-
.../core/file/rfile/RelativeKeyTest.java | 31 +-
.../server/tabletserver/TabletServer.java | 21 +-
.../test/performance/thrift/NullTserver.java | 2 +-
.../accumulo/test/ConditionalWriterTest.java | 57 +-
14 files changed, 865 insertions(+), 725 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java b/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
new file mode 100644
index 0000000..549322e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/CompressedIterators.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.util.UnsynchronizedBuffer;
+
+public class CompressedIterators {
+ private Map<String,Integer> symbolMap;
+ private List<String> symbolTable;
+ private Map<ByteSequence,IterConfig> cache;
+
+ public static class IterConfig {
+ public List<IterInfo> ssiList = new ArrayList<IterInfo>();
+ public Map<String,Map<String,String>> ssio = new HashMap<String,Map<String,String>>();
+ }
+
+ public CompressedIterators() {
+ symbolMap = new HashMap<String,Integer>();
+ symbolTable = new ArrayList<String>();
+ }
+
+ public CompressedIterators(List<String> symbols) {
+ this.symbolTable = symbols;
+ this.cache = new HashMap<ByteSequence,IterConfig>();
+ }
+
+ private int getSymbolID(String symbol) {
+ Integer id = symbolMap.get(symbol);
+ if (id == null) {
+ id = symbolTable.size();
+ symbolTable.add(symbol);
+ symbolMap.put(symbol, id);
+ }
+
+ return id;
+ }
+
+ public ByteBuffer compress(IteratorSetting[] iterators) {
+
+ UnsynchronizedBuffer.Writer out = new UnsynchronizedBuffer.Writer(iterators.length * 8);
+
+ out.writeVInt(iterators.length);
+
+ for (IteratorSetting is : iterators) {
+ out.writeVInt(getSymbolID(is.getName()));
+ out.writeVInt(getSymbolID(is.getIteratorClass()));
+ out.writeVInt(is.getPriority());
+
+ Map<String,String> opts = is.getOptions();
+ out.writeVInt(opts.size());
+
+ for (Entry<String,String> entry : opts.entrySet()) {
+ out.writeVInt(getSymbolID(entry.getKey()));
+ out.writeVInt(getSymbolID(entry.getValue()));
+ }
+ }
+
+ return out.toByteBuffer();
+
+ }
+
+ public IterConfig decompress(ByteBuffer iterators) {
+
+ ByteSequence iterKey = new ArrayByteSequence(iterators);
+ IterConfig config = cache.get(iterKey);
+ if (config != null) {
+ return config;
+ }
+
+ config = new IterConfig();
+
+ UnsynchronizedBuffer.Reader in = new UnsynchronizedBuffer.Reader(iterators);
+
+ int num = in.readVInt();
+
+ for (int i = 0; i < num; i++) {
+ String name = symbolTable.get(in.readVInt());
+ String iterClass = symbolTable.get(in.readVInt());
+ int prio = in.readVInt();
+
+ config.ssiList.add(new IterInfo(prio, iterClass, name));
+
+ int numOpts = in.readVInt();
+
+ HashMap<String,String> opts = new HashMap<String,String>();
+
+ for (int j = 0; j < numOpts; j++) {
+ String key = symbolTable.get(in.readVInt());
+ String val = symbolTable.get(in.readVInt());
+
+ opts.put(key, val);
+ }
+
+ config.ssio.put(name, opts);
+
+ }
+
+ cache.put(iterKey, config);
+ return config;
+ }
+
+ public List<String> getSymbolTable() {
+ return symbolTable;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
index b8238ac..3b6f8a5 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ConditionalWriterImpl.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.core.client.impl;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -37,14 +38,12 @@ import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.ConditionalWriter;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.TabletLocator.TabletServerMutations;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Condition;
import org.apache.accumulo.core.data.ConditionalMutation;
import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.thrift.IterInfo;
import org.apache.accumulo.core.data.thrift.TCMResult;
import org.apache.accumulo.core.data.thrift.TCMStatus;
import org.apache.accumulo.core.data.thrift.TCondition;
@@ -346,9 +345,11 @@ class ConditionalWriterImpl implements ConditionalWriter {
Map<TKeyExtent,List<TConditionalMutation>> tmutations = new HashMap<TKeyExtent,List<TConditionalMutation>>();
- convertMutations(mutations, cmidToCm, cmid, tmutations);
+ CompressedIterators compressedIters = new CompressedIterators();
+ convertMutations(mutations, cmidToCm, cmid, tmutations, compressedIters);
- List<TCMResult> tresults = client.conditionalUpdate(tinfo, credentials, ByteBufferUtil.toByteBuffers(auths.getAuthorizations()), tmutations);
+ List<TCMResult> tresults = client.conditionalUpdate(tinfo, credentials, ByteBufferUtil.toByteBuffers(auths.getAuthorizations()), tmutations,
+ compressedIters.getSymbolTable());
HashSet<KeyExtent> extentsToInvalidate = new HashSet<KeyExtent>();
@@ -406,9 +407,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
}
private void convertMutations(TabletServerMutations<QCMutation> mutations, Map<Long,CMK> cmidToCm, MutableLong cmid,
- Map<TKeyExtent,List<TConditionalMutation>> tmutations) {
-
- // TODO compress repeated iterator configurations
+ Map<TKeyExtent,List<TConditionalMutation>> tmutations, CompressedIterators compressedIters) {
for (Entry<KeyExtent,List<QCMutation>> entry : mutations.getMutations().entrySet()) {
TKeyExtent tke = entry.getKey().toThrift();
@@ -419,7 +418,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
for (QCMutation cm : condMutations) {
TMutation tm = cm.toThrift();
- List<TCondition> conditions = convertConditions(cm);
+ List<TCondition> conditions = convertConditions(cm, compressedIters);
cmidToCm.put(cmid.longValue(), new CMK(entry.getKey(), cm));
TConditionalMutation tcm = new TConditionalMutation(conditions, tm, cmid.longValue());
@@ -431,7 +430,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
}
}
- private List<TCondition> convertConditions(ConditionalMutation cm) {
+ private List<TCondition> convertConditions(ConditionalMutation cm, CompressedIterators compressedIters) {
List<TCondition> conditions = new ArrayList<TCondition>(cm.getConditions().size());
for (Condition cond : cm.getConditions()) {
@@ -443,26 +442,10 @@ class ConditionalWriterImpl implements ConditionalWriter {
hasTs = true;
}
- IteratorSetting[] iters = cond.getIterators();
-
- List<IterInfo> ssiList = new ArrayList<IterInfo>(iters.length);
- Map<String,Map<String,String>> sso = new HashMap<String,Map<String,String>>();
-
- if (iters.length == 0) {
- ssiList = Collections.emptyList();
- sso = Collections.emptyMap();
- } else {
- ssiList = new ArrayList<IterInfo>(iters.length);
- sso = new HashMap<String,Map<String,String>>();
-
- for (IteratorSetting is : iters) {
- ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(), is.getName()));
- sso.put(is.getName(), is.getOptions());
- }
- }
+ ByteBuffer iters = compressedIters.compress(cond.getIterators());
TCondition tc = new TCondition(ByteBufferUtil.toByteBuffers(cond.getFamily()), ByteBufferUtil.toByteBuffers(cond.getQualifier()),
- ByteBufferUtil.toByteBuffers(cond.getVisibility()), ts, hasTs, ByteBufferUtil.toByteBuffers(cond.getValue()), ssiList, sso);
+ ByteBufferUtil.toByteBuffers(cond.getVisibility()), ts, hasTs, ByteBufferUtil.toByteBuffers(cond.getValue()), iters);
conditions.add(tc);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
index d44a7a6..eaa61b9 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ArrayByteSequence.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.core.data;
import java.io.Serializable;
+import java.nio.ByteBuffer;
public class ArrayByteSequence extends ByteSequence implements Serializable {
@@ -48,6 +49,18 @@ public class ArrayByteSequence extends ByteSequence implements Serializable {
this(s.getBytes());
}
+ public ArrayByteSequence(ByteBuffer buffer) {
+ if (buffer.hasArray()) {
+ this.data = buffer.array();
+ this.offset = buffer.arrayOffset();
+ this.length = buffer.limit();
+ } else {
+ this.data = new byte[buffer.remaining()];
+ this.offset = 0;
+ buffer.get(data);
+ }
+ }
+
@Override
public byte byteAt(int i) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
index 00cefbf..4ac3f0c 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.accumulo.core.data.thrift.TMutation;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UnsynchronizedBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -65,121 +66,8 @@ public class Mutation implements Writable {
private byte[] data;
private int entries;
private List<byte[]> values;
-
- // created this little class instead of using ByteArrayOutput stream and DataOutputStream
- // because both are synchronized... lots of small syncs slow things down
- private static class ByteBuffer {
-
- int offset;
- byte data[] = new byte[64];
-
- private void reserve(int l) {
- if (offset + l > data.length) {
- int newSize = data.length * 2;
- while (newSize <= offset + l)
- newSize = newSize * 2;
-
- byte[] newData = new byte[newSize];
- System.arraycopy(data, 0, newData, 0, offset);
- data = newData;
- }
-
- }
-
- public void add(byte[] bytes, int off, int length) {
- reserve(length);
- System.arraycopy(bytes, off, data, offset, length);
- offset += length;
- }
-
- void add(boolean b) {
- reserve(1);
- if (b)
- data[offset++] = 1;
- else
- data[offset++] = 0;
- }
-
- public byte[] toArray() {
- byte ret[] = new byte[offset];
- System.arraycopy(data, 0, ret, 0, offset);
- return ret;
- }
-
- public void writeVLong(long i) {
- reserve(9);
- if (i >= -112 && i <= 127) {
- data[offset++] = (byte)i;
- return;
- }
-
- int len = -112;
- if (i < 0) {
- i ^= -1L; // take one's complement'
- len = -120;
- }
-
- long tmp = i;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- data[offset++] = (byte)len;
-
- len = (len < -120) ? -(len + 120) : -(len + 112);
-
- for (int idx = len; idx != 0; idx--) {
- int shiftbits = (idx - 1) * 8;
- long mask = 0xFFL << shiftbits;
- data[offset++] = (byte)((i & mask) >> shiftbits);
- }
- }
- }
-
- private static class SimpleReader {
- int offset;
- byte data[];
-
- SimpleReader(byte b[]) {
- this.data = b;
- }
-
- int readInt() {
- return (data[offset++] << 24) + ((data[offset++] & 255) << 16) + ((data[offset++] & 255) << 8) + ((data[offset++] & 255) << 0);
- }
-
- long readLong() {
- return (((long) data[offset++] << 56) + ((long) (data[offset++] & 255) << 48) + ((long) (data[offset++] & 255) << 40)
- + ((long) (data[offset++] & 255) << 32) + ((long) (data[offset++] & 255) << 24) + ((data[offset++] & 255) << 16) + ((data[offset++] & 255) << 8) + ((data[offset++] & 255) << 0));
- }
-
- void readBytes(byte b[]) {
- System.arraycopy(data, offset, b, 0, b.length);
- offset += b.length;
- }
-
- boolean readBoolean() {
- return (data[offset++] == 1);
- }
-
- long readVLong() {
- byte firstByte = data[offset++];
- int len = WritableUtils.decodeVIntSize(firstByte);
- if (len == 1) {
- return firstByte;
- }
- long i = 0;
- for (int idx = 0; idx < len-1; idx++) {
- byte b = data[offset++];
- i = i << 8;
- i = i | (b & 0xFF);
- }
- return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
- }
- }
- private ByteBuffer buffer;
+ private UnsynchronizedBuffer.Writer buffer;
private List<ColumnUpdate> updates;
@@ -205,7 +93,7 @@ public class Mutation implements Writable {
public Mutation(byte[] row, int start, int length) {
this.row = new byte[length];
System.arraycopy(row, start, this.row, 0, length);
- buffer = new ByteBuffer();
+ buffer = new UnsynchronizedBuffer.Writer();
}
public Mutation(Text row) {
@@ -445,7 +333,7 @@ public class Mutation implements Writable {
put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, true, EMPTY_BYTES);
}
- private byte[] oldReadBytes(SimpleReader in) {
+ private byte[] oldReadBytes(UnsynchronizedBuffer.Reader in) {
int len = in.readInt();
if (len == 0)
return EMPTY_BYTES;
@@ -455,7 +343,7 @@ public class Mutation implements Writable {
return bytes;
}
- private byte[] readBytes(SimpleReader in) {
+ private byte[] readBytes(UnsynchronizedBuffer.Reader in) {
int len = (int)in.readVLong();
if (len == 0)
return EMPTY_BYTES;
@@ -468,7 +356,7 @@ public class Mutation implements Writable {
public List<ColumnUpdate> getUpdates() {
serialize();
- SimpleReader in = new SimpleReader(data);
+ UnsynchronizedBuffer.Reader in = new UnsynchronizedBuffer.Reader(data);
if (updates == null) {
if (entries == 1) {
@@ -490,7 +378,7 @@ public class Mutation implements Writable {
return new ColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
}
- private ColumnUpdate deserializeColumnUpdate(SimpleReader in) {
+ private ColumnUpdate deserializeColumnUpdate(UnsynchronizedBuffer.Reader in) {
byte[] cf = readBytes(in);
byte[] cq = readBytes(in);
byte[] cv = readBytes(in);
@@ -623,8 +511,8 @@ public class Mutation implements Writable {
}
// convert data to new format
- SimpleReader din = new SimpleReader(localData);
- buffer = new ByteBuffer();
+ UnsynchronizedBuffer.Reader din = new UnsynchronizedBuffer.Reader(localData);
+ buffer = new UnsynchronizedBuffer.Writer();
for (int i = 0; i < localEntries; i++) {
byte[] cf = oldReadBytes(din);
byte[] cq = oldReadBytes(din);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java b/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
index 95de004..40a12be 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/thrift/TCondition.java
@@ -55,8 +55,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField TS_FIELD_DESC = new org.apache.thrift.protocol.TField("ts", org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.protocol.TField HAS_TIMESTAMP_FIELD_DESC = new org.apache.thrift.protocol.TField("hasTimestamp", org.apache.thrift.protocol.TType.BOOL, (short)5);
private static final org.apache.thrift.protocol.TField VAL_FIELD_DESC = new org.apache.thrift.protocol.TField("val", org.apache.thrift.protocol.TType.STRING, (short)6);
- private static final org.apache.thrift.protocol.TField SSI_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("ssiList", org.apache.thrift.protocol.TType.LIST, (short)7);
- private static final org.apache.thrift.protocol.TField SSIO_FIELD_DESC = new org.apache.thrift.protocol.TField("ssio", org.apache.thrift.protocol.TType.MAP, (short)8);
+ private static final org.apache.thrift.protocol.TField ITERATORS_FIELD_DESC = new org.apache.thrift.protocol.TField("iterators", org.apache.thrift.protocol.TType.STRING, (short)7);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@@ -70,8 +69,7 @@ import org.slf4j.LoggerFactory;
public long ts; // required
public boolean hasTimestamp; // required
public ByteBuffer val; // required
- public List<IterInfo> ssiList; // required
- public Map<String,Map<String,String>> ssio; // required
+ public ByteBuffer iterators; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
@SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -81,8 +79,7 @@ import org.slf4j.LoggerFactory;
TS((short)4, "ts"),
HAS_TIMESTAMP((short)5, "hasTimestamp"),
VAL((short)6, "val"),
- SSI_LIST((short)7, "ssiList"),
- SSIO((short)8, "ssio");
+ ITERATORS((short)7, "iterators");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@@ -109,10 +106,8 @@ import org.slf4j.LoggerFactory;
return HAS_TIMESTAMP;
case 6: // VAL
return VAL;
- case 7: // SSI_LIST
- return SSI_LIST;
- case 8: // SSIO
- return SSIO;
+ case 7: // ITERATORS
+ return ITERATORS;
default:
return null;
}
@@ -171,15 +166,8 @@ import org.slf4j.LoggerFactory;
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
tmpMap.put(_Fields.VAL, new org.apache.thrift.meta_data.FieldMetaData("val", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
- tmpMap.put(_Fields.SSI_LIST, new org.apache.thrift.meta_data.FieldMetaData("ssiList", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, IterInfo.class))));
- tmpMap.put(_Fields.SSIO, new org.apache.thrift.meta_data.FieldMetaData("ssio", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))));
+ tmpMap.put(_Fields.ITERATORS, new org.apache.thrift.meta_data.FieldMetaData("iterators", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TCondition.class, metaDataMap);
}
@@ -194,8 +182,7 @@ import org.slf4j.LoggerFactory;
long ts,
boolean hasTimestamp,
ByteBuffer val,
- List<IterInfo> ssiList,
- Map<String,Map<String,String>> ssio)
+ ByteBuffer iterators)
{
this();
this.cf = cf;
@@ -206,8 +193,7 @@ import org.slf4j.LoggerFactory;
this.hasTimestamp = hasTimestamp;
setHasTimestampIsSet(true);
this.val = val;
- this.ssiList = ssiList;
- this.ssio = ssio;
+ this.iterators = iterators;
}
/**
@@ -233,38 +219,9 @@ import org.slf4j.LoggerFactory;
this.val = org.apache.thrift.TBaseHelper.copyBinary(other.val);
;
}
- if (other.isSetSsiList()) {
- List<IterInfo> __this__ssiList = new ArrayList<IterInfo>();
- for (IterInfo other_element : other.ssiList) {
- __this__ssiList.add(new IterInfo(other_element));
- }
- this.ssiList = __this__ssiList;
- }
- if (other.isSetSsio()) {
- Map<String,Map<String,String>> __this__ssio = new HashMap<String,Map<String,String>>();
- for (Map.Entry<String, Map<String,String>> other_element : other.ssio.entrySet()) {
-
- String other_element_key = other_element.getKey();
- Map<String,String> other_element_value = other_element.getValue();
-
- String __this__ssio_copy_key = other_element_key;
-
- Map<String,String> __this__ssio_copy_value = new HashMap<String,String>();
- for (Map.Entry<String, String> other_element_value_element : other_element_value.entrySet()) {
-
- String other_element_value_element_key = other_element_value_element.getKey();
- String other_element_value_element_value = other_element_value_element.getValue();
-
- String __this__ssio_copy_value_copy_key = other_element_value_element_key;
-
- String __this__ssio_copy_value_copy_value = other_element_value_element_value;
-
- __this__ssio_copy_value.put(__this__ssio_copy_value_copy_key, __this__ssio_copy_value_copy_value);
- }
-
- __this__ssio.put(__this__ssio_copy_key, __this__ssio_copy_value);
- }
- this.ssio = __this__ssio;
+ if (other.isSetIterators()) {
+ this.iterators = org.apache.thrift.TBaseHelper.copyBinary(other.iterators);
+;
}
}
@@ -282,8 +239,7 @@ import org.slf4j.LoggerFactory;
setHasTimestampIsSet(false);
this.hasTimestamp = false;
this.val = null;
- this.ssiList = null;
- this.ssio = null;
+ this.iterators = null;
}
public byte[] getCf() {
@@ -468,77 +424,37 @@ import org.slf4j.LoggerFactory;
}
}
- public int getSsiListSize() {
- return (this.ssiList == null) ? 0 : this.ssiList.size();
- }
-
- public java.util.Iterator<IterInfo> getSsiListIterator() {
- return (this.ssiList == null) ? null : this.ssiList.iterator();
- }
-
- public void addToSsiList(IterInfo elem) {
- if (this.ssiList == null) {
- this.ssiList = new ArrayList<IterInfo>();
- }
- this.ssiList.add(elem);
+ public byte[] getIterators() {
+ setIterators(org.apache.thrift.TBaseHelper.rightSize(iterators));
+ return iterators == null ? null : iterators.array();
}
- public List<IterInfo> getSsiList() {
- return this.ssiList;
+ public ByteBuffer bufferForIterators() {
+ return iterators;
}
- public TCondition setSsiList(List<IterInfo> ssiList) {
- this.ssiList = ssiList;
+ public TCondition setIterators(byte[] iterators) {
+ setIterators(iterators == null ? (ByteBuffer)null : ByteBuffer.wrap(iterators));
return this;
}
- public void unsetSsiList() {
- this.ssiList = null;
- }
-
- /** Returns true if field ssiList is set (has been assigned a value) and false otherwise */
- public boolean isSetSsiList() {
- return this.ssiList != null;
- }
-
- public void setSsiListIsSet(boolean value) {
- if (!value) {
- this.ssiList = null;
- }
- }
-
- public int getSsioSize() {
- return (this.ssio == null) ? 0 : this.ssio.size();
- }
-
- public void putToSsio(String key, Map<String,String> val) {
- if (this.ssio == null) {
- this.ssio = new HashMap<String,Map<String,String>>();
- }
- this.ssio.put(key, val);
- }
-
- public Map<String,Map<String,String>> getSsio() {
- return this.ssio;
- }
-
- public TCondition setSsio(Map<String,Map<String,String>> ssio) {
- this.ssio = ssio;
+ public TCondition setIterators(ByteBuffer iterators) {
+ this.iterators = iterators;
return this;
}
- public void unsetSsio() {
- this.ssio = null;
+ public void unsetIterators() {
+ this.iterators = null;
}
- /** Returns true if field ssio is set (has been assigned a value) and false otherwise */
- public boolean isSetSsio() {
- return this.ssio != null;
+ /** Returns true if field iterators is set (has been assigned a value) and false otherwise */
+ public boolean isSetIterators() {
+ return this.iterators != null;
}
- public void setSsioIsSet(boolean value) {
+ public void setIteratorsIsSet(boolean value) {
if (!value) {
- this.ssio = null;
+ this.iterators = null;
}
}
@@ -592,19 +508,11 @@ import org.slf4j.LoggerFactory;
}
break;
- case SSI_LIST:
+ case ITERATORS:
if (value == null) {
- unsetSsiList();
+ unsetIterators();
} else {
- setSsiList((List<IterInfo>)value);
- }
- break;
-
- case SSIO:
- if (value == null) {
- unsetSsio();
- } else {
- setSsio((Map<String,Map<String,String>>)value);
+ setIterators((ByteBuffer)value);
}
break;
@@ -631,11 +539,8 @@ import org.slf4j.LoggerFactory;
case VAL:
return getVal();
- case SSI_LIST:
- return getSsiList();
-
- case SSIO:
- return getSsio();
+ case ITERATORS:
+ return getIterators();
}
throw new IllegalStateException();
@@ -660,10 +565,8 @@ import org.slf4j.LoggerFactory;
return isSetHasTimestamp();
case VAL:
return isSetVal();
- case SSI_LIST:
- return isSetSsiList();
- case SSIO:
- return isSetSsio();
+ case ITERATORS:
+ return isSetIterators();
}
throw new IllegalStateException();
}
@@ -735,21 +638,12 @@ import org.slf4j.LoggerFactory;
return false;
}
- boolean this_present_ssiList = true && this.isSetSsiList();
- boolean that_present_ssiList = true && that.isSetSsiList();
- if (this_present_ssiList || that_present_ssiList) {
- if (!(this_present_ssiList && that_present_ssiList))
- return false;
- if (!this.ssiList.equals(that.ssiList))
- return false;
- }
-
- boolean this_present_ssio = true && this.isSetSsio();
- boolean that_present_ssio = true && that.isSetSsio();
- if (this_present_ssio || that_present_ssio) {
- if (!(this_present_ssio && that_present_ssio))
+ boolean this_present_iterators = true && this.isSetIterators();
+ boolean that_present_iterators = true && that.isSetIterators();
+ if (this_present_iterators || that_present_iterators) {
+ if (!(this_present_iterators && that_present_iterators))
return false;
- if (!this.ssio.equals(that.ssio))
+ if (!this.iterators.equals(that.iterators))
return false;
}
@@ -829,22 +723,12 @@ import org.slf4j.LoggerFactory;
return lastComparison;
}
}
- lastComparison = Boolean.valueOf(isSetSsiList()).compareTo(typedOther.isSetSsiList());
+ lastComparison = Boolean.valueOf(isSetIterators()).compareTo(typedOther.isSetIterators());
if (lastComparison != 0) {
return lastComparison;
}
- if (isSetSsiList()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ssiList, typedOther.ssiList);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSsio()).compareTo(typedOther.isSetSsio());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSsio()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.ssio, typedOther.ssio);
+ if (isSetIterators()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.iterators, typedOther.iterators);
if (lastComparison != 0) {
return lastComparison;
}
@@ -909,19 +793,11 @@ import org.slf4j.LoggerFactory;
}
first = false;
if (!first) sb.append(", ");
- sb.append("ssiList:");
- if (this.ssiList == null) {
+ sb.append("iterators:");
+ if (this.iterators == null) {
sb.append("null");
} else {
- sb.append(this.ssiList);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("ssio:");
- if (this.ssio == null) {
- sb.append("null");
- } else {
- sb.append(this.ssio);
+ org.apache.thrift.TBaseHelper.toString(this.iterators, sb);
}
first = false;
sb.append(")");
@@ -1017,53 +893,10 @@ import org.slf4j.LoggerFactory;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
- case 7: // SSI_LIST
- if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
- {
- org.apache.thrift.protocol.TList _list78 = iprot.readListBegin();
- struct.ssiList = new ArrayList<IterInfo>(_list78.size);
- for (int _i79 = 0; _i79 < _list78.size; ++_i79)
- {
- IterInfo _elem80; // required
- _elem80 = new IterInfo();
- _elem80.read(iprot);
- struct.ssiList.add(_elem80);
- }
- iprot.readListEnd();
- }
- struct.setSsiListIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 8: // SSIO
- if (schemeField.type == org.apache.thrift.protocol.TType.MAP) {
- {
- org.apache.thrift.protocol.TMap _map81 = iprot.readMapBegin();
- struct.ssio = new HashMap<String,Map<String,String>>(2*_map81.size);
- for (int _i82 = 0; _i82 < _map81.size; ++_i82)
- {
- String _key83; // required
- Map<String,String> _val84; // required
- _key83 = iprot.readString();
- {
- org.apache.thrift.protocol.TMap _map85 = iprot.readMapBegin();
- _val84 = new HashMap<String,String>(2*_map85.size);
- for (int _i86 = 0; _i86 < _map85.size; ++_i86)
- {
- String _key87; // required
- String _val88; // required
- _key87 = iprot.readString();
- _val88 = iprot.readString();
- _val84.put(_key87, _val88);
- }
- iprot.readMapEnd();
- }
- struct.ssio.put(_key83, _val84);
- }
- iprot.readMapEnd();
- }
- struct.setSsioIsSet(true);
+ case 7: // ITERATORS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.iterators = iprot.readBinary();
+ struct.setIteratorsIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
@@ -1109,37 +942,9 @@ import org.slf4j.LoggerFactory;
oprot.writeBinary(struct.val);
oprot.writeFieldEnd();
}
- if (struct.ssiList != null) {
- oprot.writeFieldBegin(SSI_LIST_FIELD_DESC);
- {
- oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.ssiList.size()));
- for (IterInfo _iter89 : struct.ssiList)
- {
- _iter89.write(oprot);
- }
- oprot.writeListEnd();
- }
- oprot.writeFieldEnd();
- }
- if (struct.ssio != null) {
- oprot.writeFieldBegin(SSIO_FIELD_DESC);
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, struct.ssio.size()));
- for (Map.Entry<String, Map<String,String>> _iter90 : struct.ssio.entrySet())
- {
- oprot.writeString(_iter90.getKey());
- {
- oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, _iter90.getValue().size()));
- for (Map.Entry<String, String> _iter91 : _iter90.getValue().entrySet())
- {
- oprot.writeString(_iter91.getKey());
- oprot.writeString(_iter91.getValue());
- }
- oprot.writeMapEnd();
- }
- }
- oprot.writeMapEnd();
- }
+ if (struct.iterators != null) {
+ oprot.writeFieldBegin(ITERATORS_FIELD_DESC);
+ oprot.writeBinary(struct.iterators);
oprot.writeFieldEnd();
}
oprot.writeFieldStop();
@@ -1178,13 +983,10 @@ import org.slf4j.LoggerFactory;
if (struct.isSetVal()) {
optionals.set(5);
}
- if (struct.isSetSsiList()) {
+ if (struct.isSetIterators()) {
optionals.set(6);
}
- if (struct.isSetSsio()) {
- optionals.set(7);
- }
- oprot.writeBitSet(optionals, 8);
+ oprot.writeBitSet(optionals, 7);
if (struct.isSetCf()) {
oprot.writeBinary(struct.cf);
}
@@ -1203,38 +1005,15 @@ import org.slf4j.LoggerFactory;
if (struct.isSetVal()) {
oprot.writeBinary(struct.val);
}
- if (struct.isSetSsiList()) {
- {
- oprot.writeI32(struct.ssiList.size());
- for (IterInfo _iter92 : struct.ssiList)
- {
- _iter92.write(oprot);
- }
- }
- }
- if (struct.isSetSsio()) {
- {
- oprot.writeI32(struct.ssio.size());
- for (Map.Entry<String, Map<String,String>> _iter93 : struct.ssio.entrySet())
- {
- oprot.writeString(_iter93.getKey());
- {
- oprot.writeI32(_iter93.getValue().size());
- for (Map.Entry<String, String> _iter94 : _iter93.getValue().entrySet())
- {
- oprot.writeString(_iter94.getKey());
- oprot.writeString(_iter94.getValue());
- }
- }
- }
- }
+ if (struct.isSetIterators()) {
+ oprot.writeBinary(struct.iterators);
}
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, TCondition struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(8);
+ BitSet incoming = iprot.readBitSet(7);
if (incoming.get(0)) {
struct.cf = iprot.readBinary();
struct.setCfIsSet(true);
@@ -1260,44 +1039,8 @@ import org.slf4j.LoggerFactory;
struct.setValIsSet(true);
}
if (incoming.get(6)) {
- {
- org.apache.thrift.protocol.TList _list95 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.ssiList = new ArrayList<IterInfo>(_list95.size);
- for (int _i96 = 0; _i96 < _list95.size; ++_i96)
- {
- IterInfo _elem97; // required
- _elem97 = new IterInfo();
- _elem97.read(iprot);
- struct.ssiList.add(_elem97);
- }
- }
- struct.setSsiListIsSet(true);
- }
- if (incoming.get(7)) {
- {
- org.apache.thrift.protocol.TMap _map98 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.MAP, iprot.readI32());
- struct.ssio = new HashMap<String,Map<String,String>>(2*_map98.size);
- for (int _i99 = 0; _i99 < _map98.size; ++_i99)
- {
- String _key100; // required
- Map<String,String> _val101; // required
- _key100 = iprot.readString();
- {
- org.apache.thrift.protocol.TMap _map102 = new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- _val101 = new HashMap<String,String>(2*_map102.size);
- for (int _i103 = 0; _i103 < _map102.size; ++_i103)
- {
- String _key104; // required
- String _val105; // required
- _key104 = iprot.readString();
- _val105 = iprot.readString();
- _val101.put(_key104, _val105);
- }
- }
- struct.ssio.put(_key100, _val101);
- }
- }
- struct.setSsioIsSet(true);
+ struct.iterators = iprot.readBinary();
+ struct.setIteratorsIsSet(true);
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/be54c959/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
index 4ba0eb6..97001ee 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RelativeKey.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.util.UnsynchronizedBuffer;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -472,37 +473,9 @@ public class RelativeKey implements Writable {
read(in, mbseq, len);
}
- /**
- * Determines what next array size should be by rounding up to next power of two.
- *
- */
- static int nextArraySize(int i) {
- if (i < 0)
- throw new IllegalArgumentException();
-
- if (i > (1 << 30))
- return Integer.MAX_VALUE; // this is the next power of 2 minus one... a special case
-
- if (i == 0) {
- return 1;
- }
-
- // round up to next power of two
- int ret = i;
- ret--;
- ret |= ret >> 1;
- ret |= ret >> 2;
- ret |= ret >> 4;
- ret |= ret >> 8;
- ret |= ret >> 16;
- ret++;
-
- return ret;
- }
-
private static void read(DataInput in, MByteSequence mbseqDestination, int len) throws IOException {
if (mbseqDestination.getBackingArray().length < len) {
- mbseqDestination.setArray(new byte[nextArraySize(len)]);
+ mbseqDestination.setArray(new byte[UnsynchronizedBuffer.nextArraySize(len)]);
}
in.readFully(mbseqDestination.getBackingArray(), 0, len);
@@ -529,7 +502,7 @@ public class RelativeKey implements Writable {
int remainingLen = WritableUtils.readVInt(in);
int len = prefixLen + remainingLen;
if (dest.getBackingArray().length < len) {
- dest.setArray(new byte[nextArraySize(len)]);
+ dest.setArray(new byte[UnsynchronizedBuffer.nextArraySize(len)]);
}
if (prefixSource.isBackedByArray()) {
System.arraycopy(prefixSource.getBackingArray(), prefixSource.offset(), dest.getBackingArray(), 0, prefixLen);