You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/05/28 16:11:55 UTC
[1/2] incubator-ignite git commit: ignite-950: writing footer for
serializable and plain objects
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-950 [created] 0183c008c
ignite-950: writing footer for serializable and plain objects
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/1e676bcc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/1e676bcc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/1e676bcc
Branch: refs/heads/ignite-950
Commit: 1e676bcc353ccd9aaf73435257d5abe97cadc87e
Parents: f27edb3
Author: Denis Magda <dm...@gridgain.com>
Authored: Thu May 28 15:17:11 2015 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu May 28 15:17:11 2015 +0300
----------------------------------------------------------------------
.../ignite/internal/util/GridHandleTable.java | 32 +++-
.../ignite/internal/util/io/GridDataOutput.java | 7 +
.../internal/util/io/GridUnsafeDataOutput.java | 14 ++
.../optimized/OptimizedClassDescriptor.java | 58 +++++++-
.../optimized/OptimizedMarshallerUtils.java | 14 ++
.../optimized/OptimizedObjectInputStream.java | 15 ++
.../optimized/OptimizedObjectOutputStream.java | 147 +++++++++++++++++--
7 files changed, 261 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
index 05a089c..b7d8f82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
@@ -50,6 +50,9 @@ public class GridHandleTable {
/** Maps handle value -> associated object. */
private Object[] objs;
+ /** Object start offset in OutputStream. */
+ private int[] objOff;
+
/** */
private int[] spineEmpty;
@@ -70,6 +73,7 @@ public class GridHandleTable {
objs = new Object[initCap];
spineEmpty = new int[initCap];
nextEmpty = new int[initCap];
+ objOff = new int[initCap];
Arrays.fill(spineEmpty, -1);
Arrays.fill(nextEmpty, -1);
@@ -84,9 +88,10 @@ public class GridHandleTable {
* no mapping found.
*
* @param obj Object.
+ * @param startOff Object's start offset in the OutputStream.
* @return Handle.
*/
- public int lookup(Object obj) {
+ public int lookup(Object obj, int startOff) {
int idx = hash(obj) % spine.length;
if (size > 0) {
@@ -101,7 +106,7 @@ public class GridHandleTable {
if (size >= threshold)
growSpine();
- insert(obj, size, idx);
+ insert(obj, size, idx, startOff);
size++;
@@ -109,6 +114,16 @@ public class GridHandleTable {
}
/**
+ * Returns object start offset in the OutputStream.
+ *
+ * @param handle Handle ID.
+ * @return Offset.
+ */
+ public int objectOffset(int handle) {
+ return objOff[handle];
+ }
+
+ /**
* Resets table to its initial (empty) state.
*/
public void clear() {
@@ -116,6 +131,7 @@ public class GridHandleTable {
UNSAFE.copyMemory(nextEmpty, intArrOff, next, intArrOff, nextEmpty.length << 2);
Arrays.fill(objs, null);
+ Arrays.fill(objOff, 0);
size = 0;
}
@@ -134,9 +150,11 @@ public class GridHandleTable {
* @param obj Object.
* @param handle Handle.
* @param idx Index.
+ * @param startOff Object's start offset in the OutputStream.
*/
- private void insert(Object obj, int handle, int idx) {
+ private void insert(Object obj, int handle, int idx, int startOff) {
objs[handle] = obj;
+ objOff[handle] = startOff;
next[handle] = spine[idx];
spine[idx] = handle;
}
@@ -161,7 +179,7 @@ public class GridHandleTable {
int idx = hash(obj) % spine.length;
- insert(objs[i], i, idx);
+ insert(objs[i], i, idx, objOff[i]);
}
}
@@ -184,6 +202,12 @@ public class GridHandleTable {
System.arraycopy(objs, 0, newObjs, 0, size);
objs = newObjs;
+
+ int[] newObjOff = new int[newLen];
+
+ System.arraycopy(objOff, 0, newObjOff, 0, size);
+
+ objOff = newObjOff;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataOutput.java
index 579c61b..f38f91a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataOutput.java
@@ -54,6 +54,13 @@ public interface GridDataOutput extends DataOutput {
public void reset();
/**
+ * Total number of bytes already written so far.
+ *
+ * @return Number of bytes.
+ */
+ public int size();
+
+ /**
* Writes array of {@code byte}s.
*
* @param arr Array.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
index e299f9e..e3d0473 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
@@ -68,6 +68,9 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
/** Offset. */
private int off;
+ /** Total number of bytes already written. */
+ private int size;
+
/** Underlying output stream. */
private OutputStream out;
@@ -98,6 +101,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
public void bytes(byte[] bytes, int off) {
this.bytes = bytes;
this.off = off;
+ size = 0;
}
/**
@@ -107,6 +111,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
this.out = out;
off = 0;
+ size = 0;
}
/** {@inheritDoc} */
@@ -128,11 +133,17 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
return off;
}
+
/** {@inheritDoc} */
@Override public void offset(int off) {
this.off = off;
}
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return size;
+ }
+
/**
* @param size Size.
*/
@@ -175,6 +186,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
out.write(bytes, 0, size);
else
off += size;
+
+ this.size += size;
}
/** {@inheritDoc} */
@@ -259,6 +272,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
/** {@inheritDoc} */
@Override public void reset() {
off = 0;
+ size = 0;
out = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java
index 05ef534..6eb14f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java
@@ -337,6 +337,9 @@ class OptimizedClassDescriptor {
writeObjMtds = new ArrayList<>();
readObjMtds = new ArrayList<>();
List<ClassFields> fields = new ArrayList<>();
+ HashSet<String> fieldsSet = new HashSet<>();
+
+ boolean fieldsIndexingEnabled = true;
for (c = cls; c != null && !c.equals(Object.class); c = c.getSuperclass()) {
Method mtd;
@@ -379,9 +382,13 @@ class OptimizedClassDescriptor {
Map<String, Field> fieldNames = new HashMap<>();
- for (Field f : clsFields0)
+ for (Field f : clsFields0) {
fieldNames.put(f.getName(), f);
+ if (!fieldsSet.add(f.getName()))
+ fieldsIndexingEnabled = false;
+ }
+
List<FieldInfo> clsFields = new ArrayList<>(clsFields0.length);
boolean hasSerialPersistentFields = false;
@@ -408,6 +415,7 @@ class OptimizedClassDescriptor {
fieldInfo = new FieldInfo(null,
serField.getName(),
-1,
+ resolveFieldId(serField.getName()),
fieldType(serField.getType()));
}
else {
@@ -416,6 +424,7 @@ class OptimizedClassDescriptor {
fieldInfo = new FieldInfo(f,
serField.getName(),
UNSAFE.objectFieldOffset(f),
+ resolveFieldId(serField.getName()),
fieldType(serField.getType()));
}
@@ -438,8 +447,11 @@ class OptimizedClassDescriptor {
int mod = f.getModifiers();
if (!isStatic(mod) && !isTransient(mod)) {
- FieldInfo fieldInfo = new FieldInfo(f, f.getName(),
- UNSAFE.objectFieldOffset(f), fieldType(f.getType()));
+ FieldInfo fieldInfo = new FieldInfo(f,
+ f.getName(),
+ UNSAFE.objectFieldOffset(f),
+ resolveFieldId(f.getName()),
+ fieldType(f.getType()));
clsFields.add(fieldInfo);
}
@@ -459,7 +471,7 @@ class OptimizedClassDescriptor {
Collections.reverse(readObjMtds);
Collections.reverse(fields);
- this.fields = new Fields(fields);
+ this.fields = new Fields(fields, fieldsIndexingEnabled);
}
}
}
@@ -816,17 +828,21 @@ class OptimizedClassDescriptor {
/** Field name. */
private final String fieldName;
+ /** ID calculated from field's name. */
+ private final int fieldId;
+
/**
* @param field Field.
* @param name Field name.
* @param offset Field offset.
* @param type Grid optimized field type.
*/
- FieldInfo(Field field, String name, long offset, OptimizedFieldType type) {
+ FieldInfo(Field field, String name, long offset, int fieldId, OptimizedFieldType type) {
this.field = field;
fieldOffs = offset;
fieldType = type;
fieldName = name;
+ this.fieldId = fieldId;
}
/**
@@ -856,6 +872,13 @@ class OptimizedClassDescriptor {
String name() {
return fieldName;
}
+
+ /**
+ * @return Field ID.
+ */
+ int id() {
+ return fieldId;
+ }
}
/**
@@ -865,6 +888,7 @@ class OptimizedClassDescriptor {
/** Fields. */
private final List<FieldInfo> fields;
+ /** */
private final Map<String, Integer> nameToIndex;
/**
@@ -923,13 +947,17 @@ class OptimizedClassDescriptor {
/** Own fields (excluding inherited). */
private final List<Field> ownFields;
+ /** Fields indexing flag. */
+ private final boolean fieldsIndexingEnabled;
+
/**
* Creates new instance.
*
* @param fields Fields.
*/
- Fields(List<ClassFields> fields) {
+ Fields(List<ClassFields> fields, boolean fieldsIndexingEnabled) {
this.fields = fields;
+ this.fieldsIndexingEnabled = fieldsIndexingEnabled;
if (fields.isEmpty())
ownFields = null;
@@ -961,5 +989,23 @@ class OptimizedClassDescriptor {
ClassFields fields(int i) {
return fields.get(i);
}
+
+ /**
+ * Whether fields indexing is enabled for a given object or not.
+ *
+ * @return {@code true} if enabled, {@code false} otherwise.
+ */
+ boolean fieldsIndexingEnabled() {
+ return fieldsIndexingEnabled;
+ }
+
+ /**
+ * Returns a total number of hierarchy levels.
+ *
+ * @return number of hierarchy levels.
+ */
+ int hierarchyLevels() {
+ return fields.size();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
index fafdea1..6d66602 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
@@ -143,6 +143,12 @@ class OptimizedMarshallerUtils {
/** */
static final byte SERIALIZABLE = 102;
+ /** */
+ static final byte FOOTER_START = 1;
+
+ /** */
+ static final byte EMPTY_FOOTER = 2;
+
/** UTF-8 character name. */
static final Charset UTF_8 = Charset.forName("UTF-8");
@@ -229,6 +235,14 @@ class OptimizedMarshallerUtils {
}
/**
+ * @param fieldName Field name.
+ * @return Field ID.
+ */
+ static int resolveFieldId(String fieldName) {
+ return fieldName.hashCode();
+ }
+
+ /**
* Gets descriptor for provided ID.
*
* @param clsMap Class descriptors by class map.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
index 78190ea..8c825ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
@@ -518,6 +518,21 @@ class OptimizedObjectInputStream extends ObjectInputStream {
}
}
+ byte flag = in.readByte();
+
+ assert flag == EMPTY_FOOTER || flag == FOOTER_START;
+
+ if (flag == FOOTER_START) {
+ int skip = 2;
+
+ for (int i = 0; i < fields.hierarchyLevels(); i++)
+ skip += fields.fields(i).size() * 3;
+
+ skip *= 4; // all the values are integers
+
+ in.skipBytes(skip);
+ }
+
return obj;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1e676bcc/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
index 82ae129..525a2b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -64,6 +64,9 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
private OptimizedClassDescriptor.ClassFields curFields;
/** */
+ private Footer curFooter;
+
+ /** */
private PutFieldImpl curPut;
/** */
@@ -150,11 +153,16 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
*
* @param obj Object.
* @throws IOException In case of error.
+ *
+ * @return Handle ID that has already written {@code obj} or -1 if the {@code obj} has not been written before.
*/
- private void writeObject0(Object obj) throws IOException {
+ private int writeObject0(Object obj) throws IOException {
curObj = null;
curFields = null;
curPut = null;
+ curFooter = null;
+
+ int handle = -1;
if (obj == null)
writeByte(NULL);
@@ -184,7 +192,7 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
if (desc.excluded()) {
writeByte(NULL);
- return;
+ return handle;
}
Object obj0 = desc.replace(obj);
@@ -192,13 +200,11 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
if (obj0 == null) {
writeByte(NULL);
- return;
+ return handle;
}
- int handle = -1;
-
if (!desc.isPrimitive() && !desc.isEnum() && !desc.isClass())
- handle = handles.lookup(obj);
+ handle = handles.lookup(obj, out.size());
if (obj0 != obj) {
obj = obj0;
@@ -217,6 +223,8 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
desc.write(this, obj);
}
}
+
+ return handle;
}
/**
@@ -300,12 +308,15 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
@SuppressWarnings("ForLoopReplaceableByForEach")
void writeSerializable(Object obj, List<Method> mtds, OptimizedClassDescriptor.Fields fields)
throws IOException {
+ Footer footer = new Footer(fields);
+
for (int i = 0; i < mtds.size(); i++) {
Method mtd = mtds.get(i);
if (mtd != null) {
curObj = obj;
curFields = fields.fields(i);
+ curFooter = footer;
try {
mtd.invoke(obj, this);
@@ -318,8 +329,10 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
}
}
else
- writeFields(obj, fields.fields(i));
+ writeFields(obj, fields.fields(i), footer);
}
+
+ footer.write();
}
/**
@@ -451,10 +464,16 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
* @throws IOException In case of error.
*/
@SuppressWarnings("ForLoopReplaceableByForEach")
- private void writeFields(Object obj, OptimizedClassDescriptor.ClassFields fields) throws IOException {
+ private void writeFields(Object obj, OptimizedClassDescriptor.ClassFields fields, Footer footer)
+ throws IOException {
+ int size;
+ int offset;
+
for (int i = 0; i < fields.size(); i++) {
OptimizedClassDescriptor.FieldInfo t = fields.get(i);
+ offset = size = out.size();
+
switch (t.type()) {
case BYTE:
if (t.field() != null)
@@ -505,9 +524,16 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
break;
case OTHER:
- if (t.field() != null)
- writeObject0(getObject(obj, t.offset()));
+ if (t.field() != null) {
+ int handle = writeObject0(getObject(obj, t.offset()));
+
+ if (handle >= 0)
+ offset = handles.objectOffset(handle);
+ }
}
+
+ if (t.field() != null)
+ footer.put(t.id(), offset, out.size() - size);
}
}
@@ -676,7 +702,7 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
if (curObj == null)
throw new NotActiveException("Not in writeObject() call.");
- writeFields(curObj, curFields);
+ writeFields(curObj, curFields, curFooter);
}
/** {@inheritDoc} */
@@ -698,8 +724,16 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
if (curPut == null)
throw new NotActiveException("putFields() was not called.");
- for (IgniteBiTuple<OptimizedFieldType, Object> t : curPut.objs) {
- switch (t.get1()) {
+ int size;
+ int offset;
+
+ Footer footer = curPut.curFooter;
+
+ for (IgniteBiTuple<OptimizedClassDescriptor.FieldInfo, Object> t : curPut.objs) {
+
+ offset = size = out.size();
+
+ switch (t.get1().type()) {
case BYTE:
writeByte((Byte)t.get2());
@@ -741,8 +775,13 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
break;
case OTHER:
- writeObject0(t.get2());
+ int handle = writeObject0(t.get2());
+
+ if (handle >= 0)
+ offset = handles.objectOffset(handle);
}
+
+ footer.put(t.get1().id(), offset, out.size() - size);
}
}
@@ -785,8 +824,12 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
/** Fields info. */
private final OptimizedClassDescriptor.ClassFields curFields;
+
/** Values. */
- private final IgniteBiTuple<OptimizedFieldType, Object>[] objs;
+ private final IgniteBiTuple<OptimizedClassDescriptor.FieldInfo, Object>[] objs;
+
+ /** Footer. */
+ private final Footer curFooter;
/**
* @param out Output stream.
@@ -796,6 +839,7 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
this.out = out;
curFields = out.curFields;
+ curFooter = out.curFooter;
objs = new IgniteBiTuple[curFields.size()];
}
@@ -862,7 +906,78 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
OptimizedClassDescriptor.FieldInfo info = curFields.get(i);
- objs[i] = F.t(info.type(), val);
+ objs[i] = F.t(info, val);
+ }
+ }
+
+ /**
+ *
+ */
+ private class Footer {
+ /** */
+ private int[] data;
+
+ /** */
+ private int pos;
+
+ /** */
+ private int fieldsStartOff;
+
+ /**
+ * Constructor.
+ *
+ * @param fields Fields.
+ */
+ private Footer(OptimizedClassDescriptor.Fields fields) {
+ if (fields.fieldsIndexingEnabled()) {
+ int totalFooterSize = 0;
+
+ for (int i = 0; i < fields.hierarchyLevels(); i++)
+ totalFooterSize += fields.fields(i).size() * 3;
+
+ data = new int[totalFooterSize];
+
+ fieldsStartOff = out.size();
+ }
+ else
+ data = null;
+ }
+
+ /**
+ * Puts type ID and its value length to the footer.
+ *
+ * @param typeId Type ID.
+ * @param offset Start offset of an object in the marhsalled array.
+ * @param length Total number of bytes occupied by type's value.
+ */
+ private void put(int typeId, int offset, int length) {
+ if (data == null)
+ return;
+
+ data[pos++] = typeId;
+ data[pos++] = offset;
+ data[pos++] = length;
+ }
+
+ /**
+ * Writes footer content to the OutputStream.
+ *
+ * @throws IOException In case of error.
+ */
+ private void write() throws IOException {
+ if (data == null)
+ writeByte(EMPTY_FOOTER);
+ else {
+ int footerStartOff = out.size();
+
+ writeByte(FOOTER_START);
+ writeInt(fieldsStartOff);
+
+ for (int i = 0; i < data.length; i++)
+ writeInt(data[i]);
+
+ writeInt(footerStartOff);
+ }
}
}
}
[2/2] incubator-ignite git commit: ignite-950: writing footer for
serializable and plain objects
Posted by sb...@apache.org.
ignite-950: writing footer for serializable and plain objects
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/0183c008
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/0183c008
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/0183c008
Branch: refs/heads/ignite-950
Commit: 0183c008c78164448b4f0c400d52c83732e16f96
Parents: 1e676bc
Author: Denis Magda <dm...@gridgain.com>
Authored: Thu May 28 17:11:16 2015 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu May 28 17:11:16 2015 +0300
----------------------------------------------------------------------
.../ignite/internal/util/io/GridDataInput.java | 23 +++++++
.../internal/util/io/GridUnsafeDataInput.java | 70 ++++++++++++--------
.../optimized/OptimizedMarshaller.java | 28 ++++++++
.../optimized/OptimizedMarshallerUtils.java | 3 +-
.../optimized/OptimizedObjectInputStream.java | 50 +++++++++++++-
.../optimized/OptimizedObjectOutputStream.java | 6 +-
.../marshaller/GridMarshallerAbstractTest.java | 2 +-
.../optimized/OptimizedMarshallerSelfTest.java | 13 ++++
8 files changed, 161 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataInput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataInput.java
index a0c556e..81c6bcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridDataInput.java
@@ -30,12 +30,35 @@ public interface GridDataInput extends DataInput {
public void bytes(byte[] bytes, int len);
/**
+ * @param bytes Bytes.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public void bytes(byte[] bytes, int off, int len);
+
+ /**
* @param in Underlying input stream.
* @throws IOException In case of error.
*/
public void inputStream(InputStream in) throws IOException;
/**
+ * Sets offset to the new position. Supported only for {@code GridDataInput} backed by byte array.
+ *
+ * @param off Offset.
+ * @throws IOException In case of error.
+ */
+ public void offset(int off) throws IOException;
+
+ /**
+ * Size of {@code GridDataInput}. Supported only for instances backed by byte array.
+ *
+ * @return Total number of bytes available.
+ * @throws IOException In case of error.
+ */
+ public int size() throws IOException;
+
+ /**
* Resets data output.
*
* @throws IOException In case of error.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
index fb498d6..5f11768 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
@@ -71,10 +71,10 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
@GridToStringExclude
private final char[] urfCBuf = new char[CHAR_BUF_SIZE];
- /** Current offset into buf. */
+ /** Current increaseOffset into buf. */
private int pos;
- /** End offset of valid data in buf, or -1 if no more block data. */
+ /** End increaseOffset of valid data in buf, or -1 if no more block data. */
private int end = -1;
/** Bytes. */
@@ -106,18 +106,15 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
bytes(bytes, 0, len);
}
- /**
- * @param bytes Bytes.
- * @param off Offset.
- * @param len Length.
- */
- public void bytes(byte[] bytes, int off, int len) {
+ /** {@inheritDoc} */
+ @Override public void bytes(byte[] bytes, int off, int len) {
buf = bytes;
max = len;
this.off = off;
}
+
/** {@inheritDoc} */
@Override public void inputStream(InputStream in) throws IOException {
this.in = in;
@@ -125,6 +122,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
buf = inBuf;
}
+
/**
* Reads from stream to buffer. If stream is {@code null}, this method is no-op.
*
@@ -170,12 +168,28 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
}
}
+ /** {@inheritDoc} */
+ @Override public int size() throws IOException {
+ if (in != null)
+ throw new IOException("Unable to get bytes available in InputStream");
+
+ return max;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void offset(int off) throws IOException {
+ if (in != null)
+ throw new IOException("Unable to change offset in InputStream");
+
+ this.off = off;
+ }
+
/**
* @param more Bytes to move forward.
* @return Old offset value.
* @throws IOException In case of error.
*/
- private int offset(int more) throws IOException {
+ private int increaseOffset(int more) throws IOException {
int old = off;
off += more;
@@ -204,7 +218,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
byte[] arr = new byte[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(arrSize), arr, byteArrOff, arrSize);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(arrSize), arr, byteArrOff, arrSize);
return arr;
}
@@ -219,7 +233,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
short[] arr = new short[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(bytesToCp), arr, shortArrOff, bytesToCp);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(bytesToCp), arr, shortArrOff, bytesToCp);
return arr;
}
@@ -234,7 +248,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
int[] arr = new int[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(bytesToCp), arr, intArrOff, bytesToCp);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(bytesToCp), arr, intArrOff, bytesToCp);
return arr;
}
@@ -249,7 +263,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
double[] arr = new double[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(bytesToCp), arr, doubleArrOff, bytesToCp);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(bytesToCp), arr, doubleArrOff, bytesToCp);
return arr;
}
@@ -276,7 +290,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
char[] arr = new char[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(bytesToCp), arr, charArrOff, bytesToCp);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(bytesToCp), arr, charArrOff, bytesToCp);
return arr;
}
@@ -291,7 +305,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
long[] arr = new long[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(bytesToCp), arr, longArrOff, bytesToCp);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(bytesToCp), arr, longArrOff, bytesToCp);
return arr;
}
@@ -306,7 +320,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
float[] arr = new float[arrSize];
- UNSAFE.copyMemory(buf, byteArrOff + offset(bytesToCp), arr, floatArrOff, bytesToCp);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(bytesToCp), arr, floatArrOff, bytesToCp);
return arr;
}
@@ -317,14 +331,14 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
fromStream(len);
- UNSAFE.copyMemory(buf, byteArrOff + offset(len), b, byteArrOff, len);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(len), b, byteArrOff, len);
}
/** {@inheritDoc} */
@Override public void readFully(byte[] b, int off, int len) throws IOException {
fromStream(len);
- UNSAFE.copyMemory(buf, byteArrOff + offset(len), b, byteArrOff + off, len);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(len), b, byteArrOff + off, len);
}
/** {@inheritDoc} */
@@ -341,14 +355,14 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
@Override public boolean readBoolean() throws IOException {
fromStream(1);
- return UNSAFE.getBoolean(buf, byteArrOff + offset(1));
+ return UNSAFE.getBoolean(buf, byteArrOff + increaseOffset(1));
}
/** {@inheritDoc} */
@Override public byte readByte() throws IOException {
fromStream(1);
- return UNSAFE.getByte(buf, byteArrOff + offset(1));
+ return UNSAFE.getByte(buf, byteArrOff + increaseOffset(1));
}
/** {@inheritDoc} */
@@ -360,7 +374,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
@Override public short readShort() throws IOException {
fromStream(2);
- return UNSAFE.getShort(buf, byteArrOff + offset(2));
+ return UNSAFE.getShort(buf, byteArrOff + increaseOffset(2));
}
/** {@inheritDoc} */
@@ -374,7 +388,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
char v = UNSAFE.getChar(buf, byteArrOff + off);
- offset(2);
+ increaseOffset(2);
return v;
}
@@ -383,28 +397,28 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
@Override public int readInt() throws IOException {
fromStream(4);
- return UNSAFE.getInt(buf, byteArrOff + offset(4));
+ return UNSAFE.getInt(buf, byteArrOff + increaseOffset(4));
}
/** {@inheritDoc} */
@Override public long readLong() throws IOException {
fromStream(8);
- return UNSAFE.getLong(buf, byteArrOff + offset(8));
+ return UNSAFE.getLong(buf, byteArrOff + increaseOffset(8));
}
/** {@inheritDoc} */
@Override public float readFloat() throws IOException {
fromStream(4);
- return UNSAFE.getFloat(buf, byteArrOff + offset(4));
+ return UNSAFE.getFloat(buf, byteArrOff + increaseOffset(4));
}
/** {@inheritDoc} */
@Override public double readDouble() throws IOException {
fromStream(8);
- return UNSAFE.getDouble(buf, byteArrOff + offset(8));
+ return UNSAFE.getDouble(buf, byteArrOff + increaseOffset(8));
}
/** {@inheritDoc} */
@@ -433,7 +447,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
else {
int toRead = Math.min(len, max - this.off);
- UNSAFE.copyMemory(buf, byteArrOff + offset(toRead), b, byteArrOff + off, toRead);
+ UNSAFE.copyMemory(buf, byteArrOff + increaseOffset(toRead), b, byteArrOff + off, toRead);
return toRead;
}
@@ -511,7 +525,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
/**
* Reads span of UTF-encoded characters out of internal buffer
- * (starting at offset pos and ending at or before offset end),
+ * (starting at increaseOffset pos and ending at or before increaseOffset end),
* consuming no more than utfLen bytes. Appends read characters to
* sbuf. Returns the number of bytes consumed.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
index accfeb7..1b60485 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshaller.java
@@ -248,6 +248,34 @@ public class OptimizedMarshaller extends AbstractMarshaller {
}
}
+ //TODO:
+ public <T> T unmarshal(String fieldName, byte[] arr, @Nullable ClassLoader clsLdr) throws IgniteCheckedException {
+ assert arr != null && fieldName != null;
+
+ OptimizedObjectInputStream objIn = null;
+
+ try {
+ objIn = OptimizedObjectStreamRegistry.in();
+
+ objIn.context(clsMap, ctx, mapper, clsLdr != null ? clsLdr : dfltClsLdr);
+
+ objIn.in().bytes(arr, arr.length);
+
+ return (T)objIn.readField(fieldName);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to deserialize object with given class loader: " + clsLdr, e);
+ }
+ catch (ClassNotFoundException e) {
+ throw new IgniteCheckedException("Failed to find class with given class loader for unmarshalling " +
+ "(make sure same version of all classes are available on all nodes or enable peer-class-loading): " +
+ clsLdr, e);
+ }
+ finally {
+ OptimizedObjectStreamRegistry.closeIn(objIn);
+ }
+ }
+
/**
* Checks whether {@code GridOptimizedMarshaller} is able to work on the current JVM.
* <p>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
index 6d66602..dd13ac5 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java
@@ -147,7 +147,8 @@ class OptimizedMarshallerUtils {
static final byte FOOTER_START = 1;
/** */
- static final byte EMPTY_FOOTER = 2;
+ static final byte EMPTY_FOOTER = -1;
+
/** UTF-8 character name. */
static final Charset UTF_8 = Charset.forName("UTF-8");
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
index 8c825ac..f854dca 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java
@@ -518,7 +518,7 @@ class OptimizedObjectInputStream extends ObjectInputStream {
}
}
- byte flag = in.readByte();
+ byte flag = (byte)in.readInt();
assert flag == EMPTY_FOOTER || flag == FOOTER_START;
@@ -942,6 +942,54 @@ class OptimizedObjectInputStream extends ObjectInputStream {
return -1;
}
+ //TODO
+ Object readField(String fieldName) throws IOException, ClassNotFoundException {
+ byte type = in.readByte();
+
+ if (type != SERIALIZABLE)
+ return -1;
+
+ int fieldId = resolveFieldId(fieldName);
+
+ int end = in.size() - 4;
+ in.offset(end);
+
+ int footerStartOff = in.readInt();
+
+ if (footerStartOff == EMPTY_FOOTER)
+ return null; //TODO
+
+ int pos = footerStartOff;
+ in.offset(footerStartOff);
+
+ assert in.readInt() == FOOTER_START;
+ in.readInt(); //TODO: do I need this? skip fields start offset
+
+ int fieldOff = -1;
+
+ while (pos < end) {
+ int id = in.readInt();
+ int len = in.readInt(); //TODO: do I need this?
+
+ if (fieldId == id) {
+ fieldOff = in.readInt();
+ break;
+ }
+ else
+ // skip field offset
+ in.skipBytes(4);
+
+ pos += 12;
+ }
+
+ if (fieldOff > 0) {
+ in.offset(fieldOff);
+ return readObject();
+ }
+
+ return null; //TODO
+ }
+
/**
* Returns objects that were added to handles table.
* Used ONLY for test purposes.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
index 525a2b4..6c1e824 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -955,8 +955,8 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
return;
data[pos++] = typeId;
- data[pos++] = offset;
data[pos++] = length;
+ data[pos++] = offset;
}
/**
@@ -966,11 +966,11 @@ class OptimizedObjectOutputStream extends ObjectOutputStream {
*/
private void write() throws IOException {
if (data == null)
- writeByte(EMPTY_FOOTER);
+ writeInt(EMPTY_FOOTER);
else {
int footerStartOff = out.size();
- writeByte(FOOTER_START);
+ writeInt(FOOTER_START);
writeInt(fieldsStartOff);
for (int i = 0; i < data.length; i++)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java
index a89479b..b9f3535 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerAbstractTest.java
@@ -50,7 +50,7 @@ public abstract class GridMarshallerAbstractTest extends GridCommonAbstractTest
private static final String CACHE_NAME = "namedCache";
/** */
- private static Marshaller marsh;
+ protected static Marshaller marsh;
/** Closure job. */
protected IgniteInClosure<String> c1 = new IgniteInClosure<String>() {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0183c008/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java
index 7d09070..6c78506 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java
@@ -111,6 +111,19 @@ public class OptimizedMarshallerSelfTest extends GridMarshallerAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testFieldUnmarshalling() throws Exception {
+ TestObject2 obj = new TestObject2(5);
+
+ byte[] data = marshal(obj);
+
+ Integer i = ((OptimizedMarshaller)marsh).unmarshal("i", data, Thread.currentThread().getContextClassLoader());
+
+ assertEquals(obj.i, (int)i);
+ }
+
+ /**
* Class for nested execution test.
*/
private static class NestedTestObject implements Serializable {