You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/06/19 12:35:31 UTC
[45/50] incubator-ignite git commit: ignite-950: handles support in
the footer
ignite-950: handles support in the footer
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/41a821f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/41a821f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/41a821f2
Branch: refs/heads/ignite-950
Commit: 41a821f2e910f183c0bcacde8f65f9b91d7a3efd
Parents: 5bd8666
Author: Denis Magda <dm...@gridgain.com>
Authored: Fri Jun 19 12:19:15 2015 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Fri Jun 19 12:19:15 2015 +0300
----------------------------------------------------------------------
.../ignite/internal/util/GridHandleTable.java | 112 ++++++++++++++-----
.../optimized/OptimizedObjectOutputStream.java | 44 +++++---
.../optimized/ext/OptimizedMarshallerExt.java | 12 +-
.../ext/OptimizedObjectInputStreamExt.java | 54 +++++----
.../ext/OptimizedObjectOutputStreamExt.java | 46 +++++---
.../ext/OptimizedMarshallerExtSelfTest.java | 57 ++++++++++
6 files changed, 233 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/41a821f2/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
index 6b63360..639a5c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
@@ -47,11 +47,8 @@ public class GridHandleTable {
/** Maps handle value -> next candidate handle value. */
private int[] next;
- /** Handle absolute position in the output stream. */
- private int[] positions;
-
/** Maps handle value -> associated object. */
- private Object[] objs;
+ private ObjectInfo[] objs;
/** */
private int[] spineEmpty;
@@ -70,8 +67,7 @@ public class GridHandleTable {
spine = new int[initCap];
next = new int[initCap];
- objs = new Object[initCap];
- positions = new int[initCap];
+ objs = new ObjectInfo[initCap];
spineEmpty = new int[initCap];
nextEmpty = new int[initCap];
@@ -96,7 +92,7 @@ public class GridHandleTable {
if (size > 0) {
for (int i = spine[idx]; i >= 0; i = next[i])
- if (objs[i] == obj)
+ if (objs[i].object == obj)
return i;
}
@@ -106,7 +102,7 @@ public class GridHandleTable {
if (size >= threshold)
growSpine();
- insert(obj, size, idx, pos);
+ insert(new ObjectInfo(obj, pos), size, idx);
size++;
@@ -114,13 +110,31 @@ public class GridHandleTable {
}
/**
- * Returns handle absolute position in output stream.
+ * Keeps object's total len in bytes.
+ *
+ * @param obj Object.
+ * @param len Object's length.
+ */
+ public void objectLength(Object obj, int len) {
+ int idx = hash(obj) % spine.length;
+
+ if (size > 0) {
+ for (int i = spine[idx]; i >= 0; i = next[i])
+ if (objs[i].object == obj) {
+ objs[i].len = len;
+ return;
+ }
+ }
+ }
+
+ /**
+ * Returns object info for given handle.
*
* @param handle Handle.
- * @return Absolute position.
+ * @return Object info.
*/
- public int position(int handle) {
- return positions[handle];
+ public ObjectInfo objectInfo(int handle) {
+ return objs[handle];
}
/**
@@ -131,7 +145,6 @@ public class GridHandleTable {
UNSAFE.copyMemory(nextEmpty, intArrOff, next, intArrOff, nextEmpty.length << 2);
Arrays.fill(objs, null);
- Arrays.fill(positions, 0);
size = 0;
}
@@ -140,22 +153,25 @@ public class GridHandleTable {
* @return Returns objects that were added to handles table.
*/
public Object[] objects() {
- return objs;
+ Object[] objects = new Object[objs.length];
+
+ for (int i = 0; i < objs.length; i++)
+ objects[i] = objs[i];
+
+ return objects;
}
/**
* Inserts mapping object -> handle mapping into table. Assumes table
* is large enough to accommodate new mapping.
*
- * @param obj Object.
+ * @param obj Object info.
* @param handle Handle.
* @param idx Index.
- * @param pos Position in output stream.
*/
- private void insert(Object obj, int handle, int idx, int pos) {
+ private void insert(ObjectInfo obj, int handle, int idx) {
objs[handle] = obj;
next[handle] = spine[idx];
- positions[handle] = pos;
spine[idx] = handle;
}
@@ -175,11 +191,11 @@ public class GridHandleTable {
UNSAFE.copyMemory(spineEmpty, intArrOff, spine, intArrOff, spineEmpty.length << 2);
for (int i = 0; i < this.size; i++) {
- Object obj = objs[i];
+ Object obj = objs[i].object;
int idx = hash(obj) % spine.length;
- insert(objs[i], i, idx, positions[i]);
+ insert(objs[i], i, idx);
}
}
@@ -197,17 +213,11 @@ public class GridHandleTable {
Arrays.fill(nextEmpty, -1);
- Object[] newObjs = new Object[newLen];
+ ObjectInfo[] newObjs = new ObjectInfo[newLen];
System.arraycopy(objs, 0, newObjs, 0, size);
objs = newObjs;
-
- int[] newPositions = new int[newLen];
-
- UNSAFE.copyMemory(positions, intArrOff, newPositions, intArrOff, size << 2);
-
- positions = newPositions;
}
/**
@@ -219,4 +229,52 @@ public class GridHandleTable {
private int hash(Object obj) {
return System.identityHashCode(obj) & 0x7FFFFFFF;
}
+
+ /**
+ *
+ */
+ public static class ObjectInfo {
+ /** */
+ private Object object;
+
+ /** Object position in output stream. */
+ private int pos;
+
+ /** Object length. */
+ private int len;
+
+ /**
+ * Constructor.
+ *
+ * @param object Object.
+ * @param pos Object position in output stream.
+ */
+ public ObjectInfo(Object object, int pos) {
+ this.object = object;
+ this.pos = pos;
+ }
+
+ /**
+ * Gets object position in output stream.
+ *
+ * @return Position.
+ */
+ public int position() {
+ return pos;
+ }
+
+ /**
+ * Gets object length.
+ *
+ * @return Length in bytes.
+ */
+ public int length() {
+ return len;
+ }
+ }
+
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/41a821f2/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
index 90e2a85..17a29f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -154,15 +154,16 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
* @param obj Object.
* @throws IOException In case of error.
*
- * @return Handle's position in {@link #out} or -1 if the {@code obj} has not been written before.
+ * @return Instance of {@link org.apache.ignite.internal.util.GridHandleTable.ObjectInfo} if handle to an existed
+ * object is written or {@code null} otherwise.
*/
- private int writeObject0(Object obj) throws IOException {
+ private GridHandleTable.ObjectInfo writeObject0(Object obj) throws IOException {
curObj = null;
curFields = null;
curPut = null;
curFooter = null;
- int handle = -1;
+ GridHandleTable.ObjectInfo objInfo = null;
if (obj == null)
writeByte(NULL);
@@ -192,7 +193,7 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
if (desc.excluded()) {
writeByte(NULL);
- return handle;
+ return objInfo;
}
Object obj0 = desc.replace(obj);
@@ -200,9 +201,11 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
if (obj0 == null) {
writeByte(NULL);
- return handle;
+ return objInfo;
}
+ int handle = -1;
+
if (!desc.isPrimitive() && !desc.isEnum() && !desc.isClass())
handle = handles.lookup(obj, out.offset());
@@ -219,14 +222,19 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
writeByte(HANDLE);
writeInt(handle);
- handle = handles.position(handle);
+ objInfo = handles.objectInfo(handle);
}
- else
+ else {
+ int pos = out.offset();
+
desc.write(this, obj);
+
+ handles.objectLength(obj, out.offset() - pos);
+ }
}
}
- return handle;
+ return objInfo;
}
/**
@@ -546,10 +554,12 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
case OTHER:
if (t.field() != null) {
- int handle = writeObject0(getObject(obj, t.offset()));
+ GridHandleTable.ObjectInfo objInfo = writeObject0(getObject(obj, t.offset()));
- if (footer != null && handle >= 0)
- footer.putHandle(t.id(), handle);
+ if (footer != null && objInfo != null) {
+ footer.putHandle(t.id(), objInfo);
+ continue;
+ }
}
}
@@ -806,10 +816,12 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
break;
case OTHER:
- int handle = writeObject0(t.get2());
+ GridHandleTable.ObjectInfo objInfo = writeObject0(t.get2());
- if (footer != null && handle >= 0)
- footer.putHandle(t.get1().id(), handle);
+ if (footer != null && objInfo != null) {
+ footer.putHandle(t.get1().id(), objInfo);
+ continue;
+ }
}
if (footer != null) {
@@ -990,9 +1002,9 @@ public class OptimizedObjectOutputStream extends ObjectOutputStream {
* Puts handle ID for the given field ID.
*
* @param fieldId Field ID.
- * @param handlePos Handle position in output stream.
+ * @param objInfo Object's, referred by handle, info.
*/
- void putHandle(int fieldId, int handlePos);
+ void putHandle(int fieldId, GridHandleTable.ObjectInfo objInfo);
/**
* Writes footer content to the OutputStream.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/41a821f2/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExt.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExt.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExt.java
index a41a331..45506f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExt.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExt.java
@@ -35,13 +35,19 @@ public class OptimizedMarshallerExt extends OptimizedMarshaller {
static final byte EMPTY_FOOTER = -1;
/** */
- static final byte FOOTER_LEN_OFF = 4;
+ static final byte FOOTER_LEN_OFF = 2;
/** */
- static final byte VARIABLE_LEN = -1;
+ static final int FOOTER_BODY_LEN_MASK = 0x3FFFFFFF;
+
+ /** */
+ static final int FOOTER_BODY_IS_HANDLE_MASK = 0x40000000;
/** */
- static final byte NOT_A_HANDLE = -1;
+ static final byte FOOTER_BODY_HANDLE_MASK_BIT = 30;
+
+ /** */
+ static final byte VARIABLE_LEN = -1;
/** */
private OptimizedMarshallerExtMetaHandler metaHandler;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/41a821f2/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectInputStreamExt.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectInputStreamExt.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectInputStreamExt.java
index 7c8cc3c..a25dafc 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectInputStreamExt.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectInputStreamExt.java
@@ -16,7 +16,6 @@
*/
package org.apache.ignite.marshaller.optimized.ext;
-import org.apache.ignite.*;
import org.apache.ignite.internal.processors.cache.*;
import org.apache.ignite.internal.util.io.*;
import org.apache.ignite.marshaller.*;
@@ -57,10 +56,10 @@ public class OptimizedObjectInputStreamExt extends OptimizedObjectInputStream {
/** {@inheritDoc} */
@Override protected void skipFooter(Class<?> cls) throws IOException {
if (metaHandler.metadata(resolveTypeId(cls.getName(), mapper)) != null) {
- int footerLen = in.readInt();
+ short footerLen = in.readShort();
if (footerLen != EMPTY_FOOTER)
- in.skipBytes(footerLen - 4);
+ in.skipBytes(footerLen - 2);
}
}
@@ -113,7 +112,7 @@ public class OptimizedObjectInputStreamExt extends OptimizedObjectInputStream {
F field = null;
- if (range != null && range.start > 0) {
+ if (range != null && range.start >= 0) {
in.position(range.start);
if (in.readByte() == SERIALIZABLE && metaHandler.metadata(in.readInt()) != null)
@@ -163,50 +162,49 @@ public class OptimizedObjectInputStreamExt extends OptimizedObjectInputStream {
in.position(end - FOOTER_LEN_OFF);
- int footerLen = in.readInt();
+ short footerLen = in.readShort();
if (footerLen == EMPTY_FOOTER)
return null;
- // reading 'hasHandles' flag. 1 byte - additional offset to get to the flag position.
- in.position(in.position() - FOOTER_LEN_OFF - 1);
-
- boolean hasHandles = in.readBoolean();
-
- // 4 - skipping length at the beginning
- int footerOff = (end - footerLen) + 4;
+ // +2 - skipping length at the beginning
+ int footerOff = (end - footerLen) + 2;
in.position(footerOff);
int fieldOff = 0;
for (OptimizedObjectMetadata.FieldInfo info : meta.getMeta()) {
- if (info.id == fieldId) {
- int len = info.len == VARIABLE_LEN ? in.readInt() : info.len;
- int handlePos;
+ int len;
+ boolean isHandle;
- if (hasHandles && info.len == VARIABLE_LEN)
- handlePos = in.readInt();
- else
- handlePos = NOT_A_HANDLE;
+ if (info.len == VARIABLE_LEN) {
+ int fieldInfo = in.readInt();
+
+ len = fieldInfo & FOOTER_BODY_LEN_MASK;
+ isHandle = ((fieldInfo & FOOTER_BODY_IS_HANDLE_MASK) >> FOOTER_BODY_HANDLE_MASK_BIT) == 1;
+ }
+ else {
+ len = info.len;
+ isHandle = false;
+ }
- if (handlePos == NOT_A_HANDLE) {
+ if (info.id == fieldId) {
+ if (!isHandle) {
//object header len: 1 - for type, 4 - for type ID, 2 - for checksum.
fieldOff += 1 + 4 + clsNameLen + 2;
return new FieldRange(start + fieldOff, len);
}
- else {
- throw new IgniteException("UNSUPPORTED YET");
- }
+ else
+ return new FieldRange(in.readInt(), in.readInt());
}
else {
- fieldOff += info.len == VARIABLE_LEN ? in.readInt() : info.len;
+ fieldOff += len;
- if (hasHandles) {
- in.skipBytes(4);
- fieldOff += 4;
+ if (isHandle) {
+ in.skipBytes(8);
+ fieldOff += 8;
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/41a821f2/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectOutputStreamExt.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectOutputStreamExt.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectOutputStreamExt.java
index 4ef0d4a..02783a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectOutputStreamExt.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/ext/OptimizedObjectOutputStreamExt.java
@@ -17,6 +17,7 @@
package org.apache.ignite.marshaller.optimized.ext;
+import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.io.*;
import org.apache.ignite.marshaller.*;
import org.apache.ignite.marshaller.optimized.*;
@@ -78,7 +79,7 @@ public class OptimizedObjectOutputStreamExt extends OptimizedObjectOutputStream
private ArrayList<Integer> fields;
/** */
- private HashMap<Integer, Integer> handles;
+ private HashMap<Integer, GridHandleTable.ObjectInfo> handles;
/** */
private boolean hasHandles;
@@ -105,7 +106,7 @@ public class OptimizedObjectOutputStreamExt extends OptimizedObjectOutputStream
}
/** {@inheritDoc} */
- @Override public void putHandle(int fieldId, int handlePos) {
+ @Override public void putHandle(int fieldId, GridHandleTable.ObjectInfo objInfo) {
if (data == null)
return;
@@ -114,7 +115,7 @@ public class OptimizedObjectOutputStreamExt extends OptimizedObjectOutputStream
handles = new HashMap<>();
}
- handles.put(fieldId, handlePos);
+ handles.put(fieldId, objInfo);
// length of handle fields is 5 bytes.
put(fieldId, OptimizedFieldType.OTHER, 5);
@@ -125,31 +126,40 @@ public class OptimizedObjectOutputStreamExt extends OptimizedObjectOutputStream
if (data == null)
writeInt(EMPTY_FOOTER);
else {
- //9 - 4 bytes for len at the beginning, 4 bytes for len at the end, 1 byte for 'hasHandles' flag
- int footerLen = data.size() * 4 + 9;
+ int bodyEnd = out.offset();
+
+ // +4 - 2 bytes for footer len at the beginning, 2 bytes for footer len at the end.
+ short footerLen = (short)(data.size() * 4 + 4);
if (hasHandles)
- footerLen += data.size() * 4;
+ footerLen += handles.size() * 8;
- writeInt(footerLen);
+ writeShort(footerLen);
if (hasHandles) {
for (int i = 0; i < data.size(); i++) {
- writeInt(data.get(i));
-
- Integer handlePos = handles.get(fields.get(i));
-
- writeInt(handlePos == null ? NOT_A_HANDLE : handlePos);
+ GridHandleTable.ObjectInfo objInfo = handles.get(fields.get(i));
+
+ if (objInfo == null)
+ writeInt(data.get(i) & ~FOOTER_BODY_IS_HANDLE_MASK);
+ else {
+ writeInt(data.get(i) | FOOTER_BODY_IS_HANDLE_MASK);
+ writeInt(objInfo.position());
+
+ if (objInfo.length() == 0)
+ // field refers to its own object that hasn't set total length yet.
+ writeInt((bodyEnd - objInfo.position()) + footerLen);
+ else
+ writeInt(objInfo.length());
+ }
}
}
- else {
+ else
for (int fieldLen : data)
- writeInt(fieldLen);
- }
-
- writeBoolean(hasHandles);
+ // writing field len and resetting is handle mask
+ writeInt(fieldLen & ~FOOTER_BODY_IS_HANDLE_MASK);
- writeInt(footerLen);
+ writeShort(footerLen);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/41a821f2/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExtSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExtSelfTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExtSelfTest.java
index 9887b0b..2526de7 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExtSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/ext/OptimizedMarshallerExtSelfTest.java
@@ -88,6 +88,34 @@ public class OptimizedMarshallerExtSelfTest extends OptimizedMarshallerSelfTest
assertEquals(testObj.o2.i, (int)marsh.readField("i", arr, 0, arr.length, null));
}
+ /**
+ * @throws Exception In case of error.
+ */
+ public void testHandles() throws Exception {
+ OptimizedMarshallerExt marsh = (OptimizedMarshallerExt)OptimizedMarshallerExtSelfTest.marsh;
+
+ assertTrue(marsh.enableFieldsIndexing(SelfLinkObject.class));
+
+ SelfLinkObject selfLinkObject = new SelfLinkObject();
+ selfLinkObject.str1 = "Hello, world!";
+ selfLinkObject.str2 = selfLinkObject.str1;
+ selfLinkObject.link = selfLinkObject;
+
+ byte[] arr = marsh.marshal(selfLinkObject);
+
+ String str2 = marsh.readField("str2", arr, 0, arr.length, null);
+
+ assertEquals(selfLinkObject.str1, str2);
+
+ CacheOptimizedObjectImpl cacheObj = marsh.readField("link", arr, 0, arr.length, null);
+
+ arr = cacheObj.valueBytes(null);
+
+ SelfLinkObject selfLinkObject2 = marsh.unmarshal(arr, null);
+
+ assertEquals(selfLinkObject, selfLinkObject2);
+ }
+
/** */
private static class TestObject2 {
/** */
@@ -123,6 +151,9 @@ public class OptimizedMarshallerExtSelfTest extends OptimizedMarshallerSelfTest
/** The only meaningful field in the class, used for {@link #equals(Object o)} and {@link #hashCode()}. */
private final String str;
+ /** */
+ private TestObject t2;
+
/**
* @param str String to hold.
* @param i Integer.
@@ -174,4 +205,30 @@ public class OptimizedMarshallerExtSelfTest extends OptimizedMarshallerSelfTest
return true;
}
}
+
+ /**
+ *
+ */
+ private static class SelfLinkObject {
+ /** */
+ String str1;
+
+ /** */
+ String str2;
+
+ /** */
+ SelfLinkObject link;
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SelfLinkObject that = (SelfLinkObject)o;
+
+ if (str1 != null ? !str1.equals(that.str1) : that.str1 != null) return false;
+ if (str2 != null ? !str2.equals(that.str2) : that.str2 != null) return false;
+
+ return true;
+ }
+ }
}