You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2017/01/19 11:34:50 UTC
[1/2] ignite git commit: IGNITE-4219: Hadoop: fixed serialization of
long strings. This closes #1409.
Repository: ignite
Updated Branches:
refs/heads/ignite-2.0 e08b6ff48 -> 454b9769e
IGNITE-4219: Hadoop: fixed serialization of long strings. This closes #1409.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/43adf8a3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/43adf8a3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/43adf8a3
Branch: refs/heads/ignite-2.0
Commit: 43adf8a3f09c6b29fe3e70f62dbc58251d8d7cb9
Parents: f136542
Author: Ivan Veselovskiy <iv...@gridgain.com>
Authored: Thu Jan 19 14:34:23 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Jan 19 14:34:23 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/IgniteComputeImpl.java | 2 +
.../internal/igfs/common/IgfsMarshaller.java | 4 +-
.../processors/hadoop/HadoopDefaultJobInfo.java | 5 +-
.../internal/processors/igfs/IgfsFileImpl.java | 5 +-
.../internal/processors/igfs/IgfsUtils.java | 105 +++++++++++++++++++
.../ignite/internal/util/IgniteUtils.java | 70 -------------
.../internal/util/IgniteUtilsSelfTest.java | 66 ++++++++++++
7 files changed, 180 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
index 26c6797..3900c1f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComputeImpl.java
@@ -207,6 +207,8 @@ public class IgniteComputeImpl extends AsyncSupportAdapter<IgniteCompute>
/** {@inheritDoc} */
@Override public <R> R affinityCall(@NotNull Collection<String> cacheNames, Object affKey, IgniteCallable<R> job) {
+
+
A.notNull(affKey, "affKey");
A.notNull(job, "job");
A.ensure(!cacheNames.isEmpty(), "cachesNames mustn't be empty");
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/main/java/org/apache/ignite/internal/igfs/common/IgfsMarshaller.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/igfs/common/IgfsMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/igfs/common/IgfsMarshaller.java
index 859069a..4645437 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/igfs/common/IgfsMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/igfs/common/IgfsMarshaller.java
@@ -131,7 +131,7 @@ public class IgfsMarshaller {
writePath(out, req.destinationPath());
out.writeBoolean(req.flag());
out.writeBoolean(req.colocate());
- U.writeStringMap(out, req.properties());
+ IgfsUtils.writeStringMap(out, req.properties());
// Minor optimization.
if (msg.command() == AFFINITY) {
@@ -248,7 +248,7 @@ public class IgfsMarshaller {
req.destinationPath(readPath(in));
req.flag(in.readBoolean());
req.colocate(in.readBoolean());
- req.properties(U.readStringMap(in));
+ req.properties(IgfsUtils.readStringMap(in));
// Minor optimization.
if (cmd == AFFINITY) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
index ab38e4c..d4a29b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
@@ -133,7 +134,7 @@ public class HadoopDefaultJobInfo implements HadoopJobInfo, Externalizable {
out.writeBoolean(hasCombiner);
out.writeInt(numReduces);
- U.writeStringMap(out, props);
+ IgfsUtils.writeStringMap(out, props);
}
/** {@inheritDoc} */
@@ -144,7 +145,7 @@ public class HadoopDefaultJobInfo implements HadoopJobInfo, Externalizable {
hasCombiner = in.readBoolean();
numReduces = in.readInt();
- props = U.readStringMap(in);
+ props = IgfsUtils.readStringMap(in);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
index 98777be..8476382 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java
@@ -27,7 +27,6 @@ import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.Nullable;
@@ -219,7 +218,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
out.writeInt(blockSize);
out.writeLong(grpBlockSize);
out.writeLong(len);
- U.writeStringMap(out, props);
+ IgfsUtils.writeStringMap(out, props);
out.writeLong(accessTime);
out.writeLong(modificationTime);
out.writeByte(flags);
@@ -235,7 +234,7 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable, Binarylizab
blockSize = in.readInt();
grpBlockSize = in.readLong();
len = in.readLong();
- props = U.readStringMap(in);
+ props = IgfsUtils.readStringMap(in);
accessTime = in.readLong();
modificationTime = in.readLong();
flags = in.readByte();
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
index 4453dba..6763d2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
@@ -127,6 +127,9 @@ public class IgfsUtils {
/** Flag: this is a file. */
private static final byte FLAG_FILE = 0x2;
+ /** Maximum string length to be written at once. */
+ private static final int MAX_STR_LEN = 0xFFFF / 4;
+
static {
TRASH_IDS = new IgniteUuid[TRASH_CONCURRENCY];
@@ -973,4 +976,106 @@ public class IgfsUtils {
private static boolean hasFlag(byte flags, byte flag) {
return (flags & flag) == flag;
}
+
+ /**
+ * Reads string-to-string map written by {@link #writeStringMap(DataOutput, Map)}.
+ *
+ * @param in Data input.
+ * @throws IOException If write failed.
+ * @return Read result.
+ */
+ public static Map<String, String> readStringMap(DataInput in) throws IOException {
+ int size = in.readInt();
+
+ if (size == -1)
+ return null;
+ else {
+ Map<String, String> map = U.newHashMap(size);
+
+ for (int i = 0; i < size; i++)
+ map.put(readUTF(in), readUTF(in));
+
+ return map;
+ }
+ }
+
+ /**
+ * Writes string-to-string map to given data output.
+ *
+ * @param out Data output.
+ * @param map Map.
+ * @throws IOException If write failed.
+ */
+ public static void writeStringMap(DataOutput out, @Nullable Map<String, String> map) throws IOException {
+ if (map != null) {
+ out.writeInt(map.size());
+
+ for (Map.Entry<String, String> e : map.entrySet()) {
+ writeUTF(out, e.getKey());
+ writeUTF(out, e.getValue());
+ }
+ }
+ else
+ out.writeInt(-1);
+ }
+
+ /**
+ * Write UTF string which can be {@code null}.
+ *
+ * @param out Output stream.
+ * @param val Value.
+ * @throws IOException If failed.
+ */
+ public static void writeUTF(DataOutput out, @Nullable String val) throws IOException {
+ if (val == null)
+ out.writeInt(-1);
+ else {
+ out.writeInt(val.length());
+
+ if (val.length() <= MAX_STR_LEN)
+ out.writeUTF(val); // Optimized write in 1 chunk.
+ else {
+ int written = 0;
+
+ while (written < val.length()) {
+ int partLen = Math.min(val.length() - written, MAX_STR_LEN);
+
+ String part = val.substring(written, written + partLen);
+
+ out.writeUTF(part);
+
+ written += partLen;
+ }
+ }
+ }
+ }
+
+ /**
+ * Read UTF string which can be {@code null}.
+ *
+ * @param in Input stream.
+ * @return Value.
+ * @throws IOException If failed.
+ */
+ public static String readUTF(DataInput in) throws IOException {
+ int len = in.readInt(); // May be zero.
+
+ if (len < 0)
+ return null;
+ else {
+ if (len <= MAX_STR_LEN)
+ return in.readUTF();
+
+ StringBuilder sb = new StringBuilder(len);
+
+ do {
+ sb.append(in.readUTF());
+ }
+ while (sb.length() < len);
+
+ assert sb.length() == len;
+
+ return sb.toString();
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3dfb3c6..e1beb1f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -5113,76 +5113,6 @@ public abstract class IgniteUtils {
}
/**
- * Writes string-to-string map to given data output.
- *
- * @param out Data output.
- * @param map Map.
- * @throws IOException If write failed.
- */
- public static void writeStringMap(DataOutput out, @Nullable Map<String, String> map) throws IOException {
- if (map != null) {
- out.writeInt(map.size());
-
- for (Map.Entry<String, String> e : map.entrySet()) {
- writeUTFStringNullable(out, e.getKey());
- writeUTFStringNullable(out, e.getValue());
- }
- }
- else
- out.writeInt(-1);
- }
-
- /**
- * Reads string-to-string map written by {@link #writeStringMap(DataOutput, Map)}.
- *
- * @param in Data input.
- * @throws IOException If write failed.
- * @return Read result.
- */
- public static Map<String, String> readStringMap(DataInput in) throws IOException {
- int size = in.readInt();
-
- if (size == -1)
- return null;
- else {
- Map<String, String> map = U.newHashMap(size);
-
- for (int i = 0; i < size; i++)
- map.put(readUTFStringNullable(in), readUTFStringNullable(in));
-
- return map;
- }
- }
-
- /**
- * Write UTF string which can be {@code null}.
- *
- * @param out Output stream.
- * @param val Value.
- * @throws IOException If failed.
- */
- public static void writeUTFStringNullable(DataOutput out, @Nullable String val) throws IOException {
- if (val != null) {
- out.writeBoolean(true);
-
- out.writeUTF(val);
- }
- else
- out.writeBoolean(false);
- }
-
- /**
- * Read UTF string which can be {@code null}.
- *
- * @param in Input stream.
- * @return Value.
- * @throws IOException If failed.
- */
- public static String readUTFStringNullable(DataInput in) throws IOException {
- return in.readBoolean() ? in.readUTF() : null;
- }
-
- /**
* Read hash map.
*
* @param in Input.
http://git-wip-us.apache.org/repos/asf/ignite/blob/43adf8a3/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index d774065..7147d4a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -19,6 +19,10 @@ package org.apache.ignite.internal.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
@@ -46,6 +50,7 @@ import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.compute.ComputeJob;
import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
@@ -757,6 +762,67 @@ public class IgniteUtilsSelfTest extends GridCommonAbstractTest {
}
/**
+ * Composes a test String of given tlength.
+ *
+ * @param len The length.
+ * @return The String.
+ */
+ private static String composeString(int len) {
+ StringBuilder sb = new StringBuilder();
+
+ for (int i=0; i<len; i++)
+ sb.append((char)i);
+
+ String x = sb.toString();
+
+ assertEquals(len, x.length());
+
+ return x;
+ }
+
+ /**
+ * Writes the given String to a DataOutput, reads from DataInput, then checks if they are the same.
+ *
+ * @param s0 The String to check serialization for.
+ * @throws Exception On error.
+ */
+ private static void checkString(String s0) throws Exception {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput dout = new DataOutputStream(baos);
+
+ IgfsUtils.writeUTF(dout, s0);
+
+ DataInput din = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
+
+ String s1 = IgfsUtils.readUTF(din);
+
+ assertEquals(s0, s1);
+ }
+
+ /**
+ * Tests long String serialization/deserialization,
+ *
+ * @throws Exception If failed.
+ */
+ public void testLongStringWriteUTF() throws Exception {
+ checkString(null);
+ checkString("");
+
+ checkString("a");
+
+ checkString("Quick brown fox jumps over the lazy dog.");
+
+ String x = composeString(0xFFFF / 4 - 1);
+ checkString(x);
+
+ x = composeString(0xFFFF / 4);
+ checkString(x);
+
+ x = composeString(0xFFFF / 4 + 1);
+ checkString(x);
+ }
+
+ /**
* Test enum.
*/
private enum TestEnum {
[2/2] ignite git commit: Merge remote-tracking branch
'origin/ignite-2.0' into ignite-2.0
Posted by vo...@apache.org.
Merge remote-tracking branch 'origin/ignite-2.0' into ignite-2.0
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/454b9769
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/454b9769
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/454b9769
Branch: refs/heads/ignite-2.0
Commit: 454b9769e72775c5f6b44a36f0eef84bcce13bd7
Parents: 43adf8a e08b6ff
Author: devozerov <vo...@gridgain.com>
Authored: Thu Jan 19 14:34:43 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Jan 19 14:34:43 2017 +0300
----------------------------------------------------------------------
.../tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java | 37 ++-
.../TcpDiscoveryS3IpFinderAbstractSelfTest.java | 84 ++++++
...3IpFinderAwsCredentialsProviderSelfTest.java | 46 +++
...scoveryS3IpFinderAwsCredentialsSelfTest.java | 45 +++
.../s3/TcpDiscoveryS3IpFinderSelfTest.java | 79 -----
.../ignite/testsuites/IgniteS3TestSuite.java | 26 +-
.../cache/query/GridCacheQueryAdapter.java | 8 +
.../internal/processors/odbc/IgniteTypes.java | 69 +++++
.../internal/processors/odbc/OdbcTypes.java | 131 ++++++++
.../internal/processors/odbc/OdbcUtils.java | 85 ++++++
.../processors/odbc/escape/OdbcEscapeUtils.java | 52 +++-
.../utils/PlatformConfigurationUtils.java | 128 +++++++-
.../IgniteCacheQueryCacheDestroySelfTest.java | 142 +++++++++
.../odbc/OdbcEscapeSequenceSelfTest.java | 131 ++++++++
.../hadoop/impl/v2/HadoopV2TaskContext.java | 7 -
.../IgniteCacheQuerySelfTestSuite.java | 2 +
.../cpp/binary/src/impl/binary/binary_utils.cpp | 6 +-
.../cpp/common/include/ignite/common/utils.h | 8 +
.../cpp/common/os/linux/src/common/utils.cpp | 22 +-
.../cpp/common/os/win/src/common/utils.cpp | 14 +-
modules/platforms/cpp/odbc-test/Makefile.am | 1 +
.../odbc-test/include/sql_test_suite_fixture.h | 13 +
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 1 +
.../project/vs/odbc-test.vcxproj.filters | 3 +
.../cpp/odbc-test/src/api_robustness_test.cpp | 2 +-
.../src/sql_aggregate_functions_test.cpp | 4 +-
.../src/sql_esc_convert_function_test.cpp | 160 ++++++++++
.../odbc-test/src/sql_test_suite_fixture.cpp | 52 +++-
.../cpp/odbc-test/src/sql_types_test.cpp | 131 +++++++-
.../odbc/src/app/application_data_buffer.cpp | 58 +++-
.../platforms/cpp/odbc/src/app/parameter.cpp | 4 +-
.../cpp/odbc/src/config/connection_info.cpp | 260 ++++++++++++++--
.../Apache.Ignite.Core.Tests.csproj | 3 +
.../Binary/BinaryBuilderSelfTest.cs | 159 ++++++----
.../BinaryBuilderSelfTestArrayIdentity.cs | 34 +++
.../Binary/BinaryEqualityComparerTest.cs | 279 +++++++++++++++++
.../Binary/IO/BinaryStreamsTest.cs | 19 ++
.../Cache/CacheConfigurationTest.cs | 5 +-
.../Cache/Query/CacheDmlQueriesTest.cs | 296 +++++++++++++++++++
.../IgniteConfigurationSerializerTest.cs | 46 ++-
.../IgniteConfigurationTest.cs | 28 ++
.../Apache.Ignite.Core.csproj | 5 +
.../Binary/BinaryArrayEqualityComparer.cs | 149 ++++++++++
.../Binary/BinaryConfiguration.cs | 24 ++
.../Binary/BinaryTypeConfiguration.cs | 14 +
.../Cache/Configuration/QueryEntity.cs | 33 ++-
.../Cache/Configuration/QueryField.cs | 6 +
.../Apache.Ignite.Core/IgniteConfiguration.cs | 85 ++++--
.../IgniteConfigurationSection.xsd | 19 ++
.../Apache.Ignite.Core/Impl/Binary/Binary.cs | 28 +-
.../Binary/BinaryEqualityComparerSerializer.cs | 99 +++++++
.../Impl/Binary/BinaryFieldEqualityComparer.cs | 138 +++++++++
.../Impl/Binary/BinaryFullTypeDescriptor.cs | 21 +-
.../Impl/Binary/BinaryObject.cs | 31 +-
.../Impl/Binary/BinaryObjectBuilder.cs | 62 +++-
.../Impl/Binary/BinaryObjectHeader.cs | 21 +-
.../Impl/Binary/BinaryObjectSchemaHolder.cs | 22 ++
.../Binary/BinarySurrogateTypeDescriptor.cs | 6 +
.../Impl/Binary/BinarySystemHandlers.cs | 6 +-
.../Impl/Binary/BinaryWriter.cs | 11 +-
.../Impl/Binary/DateTimeHolder.cs | 35 ++-
.../Impl/Binary/IBinaryEqualityComparer.cs | 53 ++++
.../Impl/Binary/IBinaryTypeDescriptor.cs | 5 +
.../Impl/Binary/Io/BinaryHeapStream.cs | 9 +
.../Impl/Binary/Io/BinaryStreamBase.cs | 13 +
.../Impl/Binary/Io/IBinaryStream.cs | 11 +-
.../Impl/Binary/Io/IBinaryStreamProcessor.cs | 36 +++
.../Impl/Binary/Marshaller.cs | 22 +-
.../Impl/Binary/SerializableObjectHolder.cs | 16 +
.../Common/IgniteConfigurationXmlSerializer.cs | 5 +-
.../Impl/Memory/PlatformMemoryStream.cs | 16 +
71 files changed, 3390 insertions(+), 291 deletions(-)
----------------------------------------------------------------------