You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2016/12/22 15:15:41 UTC
[01/50] [abbrv] ignite git commit: ignite-4154 Fixed node version
check for compression feature usage
Repository: ignite
Updated Branches:
refs/heads/master 41dddb87d -> 1b2afbb8d
ignite-4154 Fixed node version check for compression feature usage
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/671a77a2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/671a77a2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/671a77a2
Branch: refs/heads/master
Commit: 671a77a2d81cac401765dddf25f30fba4e4ab17f
Parents: bbaa79a
Author: sboikov <sb...@gridgain.com>
Authored: Thu Dec 8 12:56:46 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Dec 8 12:56:46 2016 +0300
----------------------------------------------------------------------
.../GridCachePartitionExchangeManager.java | 23 +++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/671a77a2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index f04a6ce..7f11dc4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -826,7 +826,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
break;
}
- else if (node.version().compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessage.PART_MAP_COMPRESS_SINCE) < 0)
+ else if (!canUsePartitionMapCompression(node))
compress = false;
}
}
@@ -964,8 +964,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
boolean clientOnlyExchange,
boolean sndCounters)
{
- boolean compress =
- targetNode.version().compareToIgnoreTimestamp(GridDhtPartitionsSingleMessage.PART_MAP_COMPRESS_SINCE) >= 0;
+ boolean compress = canUsePartitionMapCompression(targetNode);
GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(exchangeId,
clientOnlyExchange,
@@ -1555,6 +1554,24 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
/**
+ * @param node Target node.
+ * @return {@code True} if can use compression for partition map messages.
+ */
+ @SuppressWarnings("SimplifiableIfStatement")
+ private boolean canUsePartitionMapCompression(ClusterNode node) {
+ IgniteProductVersion ver = node.version();
+
+ if (ver.compareToIgnoreTimestamp(GridDhtPartitionsAbstractMessage.PART_MAP_COMPRESS_SINCE) >= 0) {
+ if (ver.minor() == 7 && ver.maintenance() < 4)
+ return false;
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
* Exchange future thread. All exchanges happen only by one thread and next
* exchange will not start until previous one completes.
*/
[15/50] [abbrv] ignite git commit: IGNITE-4421: Added BinaryObject
handling in ODBC. Added test.
Posted by sb...@apache.org.
IGNITE-4421: Added BinaryObject handling in ODBC. Added test.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bf1770bc
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bf1770bc
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bf1770bc
Branch: refs/heads/master
Commit: bf1770bcebfb17d36ac90e157373901b5a181c05
Parents: 83710a9
Author: Igor Sapego <is...@gridgain.com>
Authored: Wed Dec 14 12:26:27 2016 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Wed Dec 14 12:26:27 2016 +0300
----------------------------------------------------------------------
.../cpp/odbc-test/config/queries-test.xml | 37 ++++++
.../platforms/cpp/odbc-test/include/Makefile.am | 1 +
.../cpp/odbc-test/include/complex_type.h | 122 +++++++++++++++++++
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 1 +
.../project/vs/odbc-test.vcxproj.filters | 3 +
.../cpp/odbc-test/src/queries_test.cpp | 101 ++++++++++++---
modules/platforms/cpp/odbc/src/column.cpp | 41 ++++---
modules/platforms/cpp/odbc/src/type_traits.cpp | 3 +
8 files changed, 277 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc-test/config/queries-test.xml
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/config/queries-test.xml b/modules/platforms/cpp/odbc-test/config/queries-test.xml
index f511c7d..906fadf 100644
--- a/modules/platforms/cpp/odbc-test/config/queries-test.xml
+++ b/modules/platforms/cpp/odbc-test/config/queries-test.xml
@@ -85,6 +85,43 @@
</list>
</property>
</bean>
+
+ <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ <property name="name" value="cache2"/>
+ <property name="cacheMode" value="PARTITIONED"/>
+ <property name="atomicityMode" value="TRANSACTIONAL"/>
+ <property name="writeSynchronizationMode" value="FULL_SYNC"/>
+
+ <!-- Configure type metadata to enable queries. -->
+ <property name="queryEntities">
+ <list>
+ <bean class="org.apache.ignite.cache.QueryEntity">
+ <property name="keyType" value="java.lang.Long"/>
+ <property name="valueType" value="ComplexType"/>
+
+ <property name="fields">
+ <map>
+ <entry key="i32Field" value="java.lang.Integer"/>
+ <entry key="objField" value="TestObject"/>
+ <entry key="strField" value="java.lang.String"/>
+ </map>
+ </property>
+
+ <property name="keyFields">
+ <list></list>
+ </property>
+
+ <property name="indexes">
+ <list>
+ <bean class="org.apache.ignite.cache.QueryIndex">
+ <constructor-arg value="i32Field"/>
+ </bean>
+ </list>
+ </property>
+ </bean>
+ </list>
+ </property>
+ </bean>
</list>
</property>
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc-test/include/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/include/Makefile.am b/modules/platforms/cpp/odbc-test/include/Makefile.am
index 832103c..d13e7b6 100644
--- a/modules/platforms/cpp/odbc-test/include/Makefile.am
+++ b/modules/platforms/cpp/odbc-test/include/Makefile.am
@@ -19,6 +19,7 @@ ACLOCAL_AMFLAGS =-I m4
noinst_HEADERS = \
teamcity/teamcity_messages.h \
+ complex_type.h \
test_type.h \
test_utils.h \
sql_test_suite_fixture.h
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc-test/include/complex_type.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/include/complex_type.h b/modules/platforms/cpp/odbc-test/include/complex_type.h
new file mode 100644
index 0000000..a84b033
--- /dev/null
+++ b/modules/platforms/cpp/odbc-test/include/complex_type.h
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_TEST_COMPLEX_TYPE
+#define _IGNITE_ODBC_TEST_COMPLEX_TYPE
+
+#include <string>
+
+#include "ignite/ignite.h"
+#include "ignite/ignition.h"
+
+namespace ignite
+{
+ struct TestObject
+ {
+ TestObject() :
+ f1(412),
+ f2("Lorem ipsum")
+ {
+ // No-op.
+ }
+
+ int32_t f1;
+ std::string f2;
+ };
+
+ struct ComplexType
+ {
+ ComplexType() :
+ i32Field(0)
+ {
+ // No-op.
+ }
+
+ int32_t i32Field;
+ TestObject objField;
+ std::string strField;
+ };
+}
+
+namespace ignite
+{
+ namespace binary
+ {
+
+ IGNITE_BINARY_TYPE_START(ignite::TestObject)
+
+ typedef ignite::TestObject TestObject;
+
+ IGNITE_BINARY_GET_TYPE_ID_AS_HASH(TestObject)
+ IGNITE_BINARY_GET_TYPE_NAME_AS_IS(TestObject)
+ IGNITE_BINARY_GET_FIELD_ID_AS_HASH
+ IGNITE_BINARY_GET_HASH_CODE_ZERO(TestObject)
+ IGNITE_BINARY_IS_NULL_FALSE(TestObject)
+ IGNITE_BINARY_GET_NULL_DEFAULT_CTOR(TestObject)
+
+ void Write(BinaryWriter& writer, TestObject obj)
+ {
+ writer.WriteInt32("f1", obj.f1);
+ writer.WriteString("f2", obj.f2);
+ }
+
+ TestObject Read(BinaryReader& reader)
+ {
+ TestObject obj;
+
+ obj.f1 = reader.ReadInt32("f1");
+ obj.f2 = reader.ReadString("f2");
+
+ return obj;
+ }
+
+ IGNITE_BINARY_TYPE_END
+
+ IGNITE_BINARY_TYPE_START(ignite::ComplexType)
+
+ typedef ignite::ComplexType ComplexType;
+
+ IGNITE_BINARY_GET_TYPE_ID_AS_HASH(ComplexType)
+ IGNITE_BINARY_GET_TYPE_NAME_AS_IS(ComplexType)
+ IGNITE_BINARY_GET_FIELD_ID_AS_HASH
+ IGNITE_BINARY_GET_HASH_CODE_ZERO(ComplexType)
+ IGNITE_BINARY_IS_NULL_FALSE(ComplexType)
+ IGNITE_BINARY_GET_NULL_DEFAULT_CTOR(ComplexType)
+
+ void Write(BinaryWriter& writer, ComplexType obj)
+ {
+ writer.WriteInt32("i32Field", obj.i32Field);
+ writer.WriteObject("objField", obj.objField);
+ writer.WriteString("strField", obj.strField);
+ }
+
+ ComplexType Read(BinaryReader& reader)
+ {
+ ComplexType obj;
+
+ obj.i32Field = reader.ReadInt32("i32Field");
+ obj.objField = reader.ReadObject<TestObject>("objField");
+ obj.strField = reader.ReadString("strField");
+
+ return obj;
+ }
+
+ IGNITE_BINARY_TYPE_END
+ }
+};
+
+#endif // _IGNITE_ODBC_TEST_COMPLEX_TYPE
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
index 91603dc..753ae4c 100644
--- a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
+++ b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
@@ -186,6 +186,7 @@
<ClCompile Include="..\..\src\utility_test.cpp" />
</ItemGroup>
<ItemGroup>
+ <ClInclude Include="..\..\include\complex_type.h" />
<ClInclude Include="..\..\include\sql_test_suite_fixture.h" />
<ClInclude Include="..\..\include\teamcity\teamcity_messages.h" />
<ClInclude Include="..\..\include\test_type.h" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
index eef6abb..bedceaa 100644
--- a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
+++ b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
@@ -129,6 +129,9 @@
<ClInclude Include="..\..\include\sql_test_suite_fixture.h">
<Filter>Code</Filter>
</ClInclude>
+ <ClInclude Include="..\..\include\complex_type.h">
+ <Filter>Code\CacheTypes</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\..\config\queries-test.xml">
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc-test/src/queries_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 554cee2..52b885d 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -37,6 +37,7 @@
#include "ignite/impl/binary/binary_utils.h"
#include "test_type.h"
+#include "complex_type.h"
#include "test_utils.h"
using namespace ignite;
@@ -150,11 +151,17 @@ struct QueriesTestSuiteFixture
/**
* Constructor.
*/
- QueriesTestSuiteFixture() : testCache(0), env(NULL), dbc(NULL), stmt(NULL)
+ QueriesTestSuiteFixture() :
+ cache1(0),
+ cache2(0),
+ env(NULL),
+ dbc(NULL),
+ stmt(NULL)
{
grid = StartNode("NodeMain", "queries-test.xml");
- testCache = grid.GetCache<int64_t, TestType>("cache");
+ cache1 = grid.GetCache<int64_t, TestType>("cache");
+ cache2 = grid.GetCache<int64_t, ComplexType>("cache2");
}
/**
@@ -180,8 +187,8 @@ struct QueriesTestSuiteFixture
TestType in2(8, 7, 6, 5, "4", 3.0f, 2.0, false, Guid(1, 0), BinaryUtils::MakeDateGmt(1976, 1, 12),
BinaryUtils::MakeTimestampGmt(1978, 8, 21, 23, 13, 45, 456));
- testCache.Put(1, in1);
- testCache.Put(2, in2);
+ cache1.Put(1, in1);
+ cache1.Put(2, in2);
const size_t columnsCnt = 11;
@@ -362,8 +369,11 @@ struct QueriesTestSuiteFixture
/** Node started during the test. */
Ignite grid;
- /** Test cache instance. */
- Cache<int64_t, TestType> testCache;
+ /** Frist cache instance. */
+ Cache<int64_t, TestType> cache1;
+
+ /** Second cache instance. */
+ Cache<int64_t, ComplexType> cache2;
/** ODBC Environment. */
SQLHENV env;
@@ -444,8 +454,8 @@ BOOST_AUTO_TEST_CASE(TestTwoRowsString)
TestType in2(8, 7, 6, 5, "4", 3.0f, 2.0, false, Guid(1, 0), BinaryUtils::MakeDateGmt(1976, 1, 12),
BinaryUtils::MakeTimestampGmt(1978, 8, 21, 23, 13, 45, 999999999));
- testCache.Put(1, in1);
- testCache.Put(2, in2);
+ cache1.Put(1, in1);
+ cache1.Put(2, in2);
const size_t columnsCnt = 11;
@@ -541,7 +551,7 @@ BOOST_AUTO_TEST_CASE(TestOneRowString)
TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
- testCache.Put(1, in);
+ cache1.Put(1, in);
const size_t columnsCnt = 11;
@@ -607,7 +617,7 @@ BOOST_AUTO_TEST_CASE(TestOneRowStringLen)
TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), BinaryUtils::MakeDateGmt(1987, 6, 5),
BinaryUtils::MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456));
- testCache.Put(1, in);
+ cache1.Put(1, in);
const size_t columnsCnt = 11;
@@ -649,6 +659,63 @@ BOOST_AUTO_TEST_CASE(TestOneRowStringLen)
BOOST_CHECK(ret == SQL_NO_DATA);
}
+BOOST_AUTO_TEST_CASE(TestOneRowObject)
+{
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache2");
+
+ SQLRETURN ret;
+
+ ComplexType obj;
+
+ obj.i32Field = 123;
+ obj.strField = "Some string";
+
+ obj.objField.f1 = 54321;
+ obj.objField.f2 = "Hello Ignite";
+
+ cache2.Put(1, obj);
+
+ int64_t column1 = 0;
+ int8_t column2[ODBC_BUFFER_SIZE] = { 0 };
+ char column3[ODBC_BUFFER_SIZE] = { 0 };
+
+ SQLLEN column1Len = sizeof(column1);
+ SQLLEN column2Len = sizeof(column2);
+ SQLLEN column3Len = sizeof(column3);
+
+ // Binding columns.
+ ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &column1, column1Len, &column1Len);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ ret = SQLBindCol(stmt, 2, SQL_C_BINARY, &column2, column2Len, &column2Len);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ ret = SQLBindCol(stmt, 3, SQL_C_CHAR, &column3, column3Len, &column3Len);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ SQLCHAR request[] = "SELECT i32Field, objField, strField FROM ComplexType";
+
+ ret = SQLExecDirect(stmt, request, SQL_NTS);
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ ret = SQLFetch(stmt);
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ BOOST_CHECK_EQUAL(column1, obj.i32Field);
+ BOOST_CHECK_EQUAL(column3, obj.strField);
+
+ ret = SQLFetch(stmt);
+ BOOST_CHECK(ret == SQL_NO_DATA);
+}
+
BOOST_AUTO_TEST_CASE(TestDataAtExecution)
{
Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
@@ -661,8 +728,8 @@ BOOST_AUTO_TEST_CASE(TestDataAtExecution)
TestType in2(8, 7, 6, 5, "4", 3.0f, 2.0, false, Guid(1, 0), BinaryUtils::MakeDateGmt(1976, 1, 12),
BinaryUtils::MakeTimestampGmt(1978, 8, 21, 23, 13, 45, 999999999));
- testCache.Put(1, in1);
- testCache.Put(2, in2);
+ cache1.Put(1, in1);
+ cache1.Put(2, in2);
const size_t columnsCnt = 11;
@@ -785,9 +852,9 @@ BOOST_AUTO_TEST_CASE(TestNullFields)
inNull.allNulls = true;
- testCache.Put(1, in);
- testCache.Put(2, inNull);
- testCache.Put(3, in);
+ cache1.Put(1, in);
+ cache1.Put(2, inNull);
+ cache1.Put(3, in);
const size_t columnsCnt = 10;
@@ -900,7 +967,7 @@ BOOST_AUTO_TEST_CASE(TestDistributedJoins)
entry.i32Field = i;
entry.i64Field = entriesNum - i - 1;
- testCache.Put(i, entry);
+ cache1.Put(i, entry);
}
Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache");
@@ -974,7 +1041,7 @@ BOOST_AUTO_TEST_CASE(TestDistributedJoinsWithOldVersion)
entry.i32Field = i;
entry.i64Field = entriesNum - i - 1;
- testCache.Put(i, entry);
+ cache1.Put(i, entry);
}
Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;CACHE=cache;DISTRIBUTED_JOINS=true;PROTOCOL_VERSION=1.6.0");
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc/src/column.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/column.cpp b/modules/platforms/cpp/odbc/src/column.cpp
index b076a12..3e6bbca 100644
--- a/modules/platforms/cpp/odbc/src/column.cpp
+++ b/modules/platforms/cpp/odbc/src/column.cpp
@@ -31,24 +31,32 @@ namespace
int8_t hdr = stream.ReadInt8();
- if (hdr != IGNITE_HDR_FULL)
- return false;
-
- int8_t protoVer = stream.ReadInt8();
+ switch (hdr)
+ {
+ case IGNITE_TYPE_BINARY:
+ {
+ // Header field + Length field + Object itself + Offset field
+ len = 1 + 4 + stream.ReadInt32() + 4;
- if (protoVer != IGNITE_PROTO_VER)
- return false;
+ break;
+ }
+
+ case IGNITE_TYPE_OBJECT:
+ {
+ int8_t protoVer = stream.ReadInt8();
- // Skipping flags
- stream.ReadInt16();
+ if (protoVer != IGNITE_PROTO_VER)
+ return false;
- // Skipping typeId
- stream.ReadInt32();
+ // Skipping flags, typeId and hash code
+ len = stream.ReadInt32(stream.Position() + 2 + 4 + 4);
- // Skipping hash code
- stream.ReadInt32();
+ break;
+ }
- len = stream.ReadInt32();
+ default:
+ return false;
+ }
return true;
}
@@ -238,7 +246,8 @@ namespace ignite
break;
}
- case IGNITE_HDR_FULL:
+ case IGNITE_TYPE_BINARY:
+ case IGNITE_TYPE_OBJECT:
{
int32_t len;
@@ -284,6 +293,7 @@ namespace ignite
default:
{
// This is a fail case.
+ assert(false);
return;
}
}
@@ -413,7 +423,8 @@ namespace ignite
break;
}
- case IGNITE_HDR_FULL:
+ case IGNITE_TYPE_BINARY:
+ case IGNITE_TYPE_OBJECT:
{
int32_t len;
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf1770bc/modules/platforms/cpp/odbc/src/type_traits.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/type_traits.cpp b/modules/platforms/cpp/odbc/src/type_traits.cpp
index fced2e1..643e1b4 100644
--- a/modules/platforms/cpp/odbc/src/type_traits.cpp
+++ b/modules/platforms/cpp/odbc/src/type_traits.cpp
@@ -299,12 +299,14 @@ namespace ignite
return IGNITE_ODBC_C_TYPE_WCHAR;
case SQL_C_SSHORT:
+ case SQL_C_SHORT:
return IGNITE_ODBC_C_TYPE_SIGNED_SHORT;
case SQL_C_USHORT:
return IGNITE_ODBC_C_TYPE_UNSIGNED_SHORT;
case SQL_C_SLONG:
+ case SQL_C_LONG:
return IGNITE_ODBC_C_TYPE_SIGNED_LONG;
case SQL_C_ULONG:
@@ -320,6 +322,7 @@ namespace ignite
return IGNITE_ODBC_C_TYPE_BIT;
case SQL_C_STINYINT:
+ case SQL_C_TINYINT:
return IGNITE_ODBC_C_TYPE_SIGNED_TINYINT;
case SQL_C_UTINYINT:
[43/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/community/ignite-1.7.4' into ignite-1.8.2
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/community/ignite-1.7.4' into ignite-1.8.2
# Conflicts:
# modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
# modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
# modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
# modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
# modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
# modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
# modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/64247b92
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/64247b92
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/64247b92
Branch: refs/heads/master
Commit: 64247b9228451e46abb8029e09c7fc6ed4e16d2d
Parents: 147277d 8dd4ada
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 19 15:54:39 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 19 15:54:39 2016 +0300
----------------------------------------------------------------------
.../store/cassandra/CassandraCacheStore.java | 9 +-
.../store/cassandra/datasource/DataSource.java | 9 +
.../rest/RestProcessorMultiStartSelfTest.java | 48 +-
.../java/org/apache/ignite/IgniteServices.java | 16 +
.../apache/ignite/IgniteSystemProperties.java | 6 +
.../rendezvous/RendezvousAffinityFunction.java | 80 ++-
.../ignite/cache/store/CacheStoreAdapter.java | 6 +
.../cache/store/jdbc/CacheJdbcPojoStore.java | 19 +-
.../store/jdbc/JdbcTypesDefaultTransformer.java | 112 ++--
.../apache/ignite/internal/IgniteKernal.java | 28 +-
.../ignite/internal/IgniteServicesImpl.java | 9 +-
.../internal/binary/BinaryClassDescriptor.java | 12 +-
.../ignite/internal/binary/BinaryUtils.java | 10 +-
.../binary/builder/BinaryObjectBuilderImpl.java | 11 +-
.../discovery/GridDiscoveryManager.java | 118 +---
.../affinity/GridAffinityProcessor.java | 2 +-
.../processors/cache/CacheLockCandidates.java | 42 ++
.../cache/CacheLockCandidatesList.java | 71 +++
.../cache/CacheStoreBalancingWrapper.java | 6 +
.../processors/cache/GridCacheAdapter.java | 8 +-
.../processors/cache/GridCacheEntryEx.java | 3 +-
.../cache/GridCacheLoaderWriterStore.java | 6 +
.../processors/cache/GridCacheMapEntry.java | 117 +++-
.../processors/cache/GridCacheMvcc.java | 376 +++++++----
.../processors/cache/GridCacheMvccCallback.java | 4 +-
.../cache/GridCacheMvccCandidate.java | 80 +--
.../processors/cache/GridCacheMvccManager.java | 19 +-
.../GridCachePartitionExchangeManager.java | 157 ++---
.../processors/cache/GridCachePreloader.java | 11 +-
.../cache/GridCachePreloaderAdapter.java | 5 +-
.../processors/cache/GridCacheProcessor.java | 10 +-
.../processors/cache/GridCacheUtils.java | 17 -
.../binary/CacheObjectBinaryProcessorImpl.java | 3 +-
.../CacheDataStructuresManager.java | 6 +-
.../distributed/GridDistributedCacheEntry.java | 303 +++------
.../dht/GridClientPartitionTopology.java | 120 ++--
.../distributed/dht/GridDhtCacheEntry.java | 32 +-
.../distributed/dht/GridDhtLockFuture.java | 34 +-
.../dht/GridDhtPartitionTopology.java | 28 +-
.../dht/GridDhtPartitionTopologyImpl.java | 284 +++++----
.../dht/GridDhtTransactionalCacheAdapter.java | 1 -
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +-
.../colocated/GridDhtColocatedLockFuture.java | 8 +-
.../dht/preloader/GridDhtPartitionDemander.java | 230 ++++---
.../dht/preloader/GridDhtPartitionFullMap.java | 18 +-
.../GridDhtPartitionsExchangeFuture.java | 56 +-
.../dht/preloader/GridDhtPreloader.java | 9 +-
.../distributed/near/GridNearCacheEntry.java | 44 +-
.../distributed/near/GridNearLockFuture.java | 3 +-
.../near/GridNearTransactionalCache.java | 5 +-
.../cache/local/GridLocalCacheEntry.java | 173 ++----
.../cache/local/GridLocalLockFuture.java | 2 +-
.../cache/query/GridCacheQueryManager.java | 22 +-
.../cache/transactions/IgniteTxHandler.java | 2 +-
.../cache/transactions/IgniteTxManager.java | 5 +-
.../closure/GridClosureProcessor.java | 31 +-
.../internal/processors/job/GridJobWorker.java | 76 ++-
.../processors/odbc/OdbcRequestHandler.java | 14 +-
.../platform/PlatformContextImpl.java | 2 +-
.../dotnet/PlatformDotNetCacheStore.java | 11 +
.../platform/services/PlatformServices.java | 2 +-
.../platform/utils/PlatformUtils.java | 28 +
.../processors/rest/GridRestProcessor.java | 15 +
.../service/GridServiceProcessor.java | 15 +-
.../processors/service/GridServiceProxy.java | 18 +-
.../processors/task/GridTaskWorker.java | 7 +
.../internal/visor/query/VisorQueryJob.java | 2 +-
.../ignite/marshaller/jdk/JdkMarshaller.java | 4 +-
.../optimized/OptimizedMarshaller.java | 8 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 41 +-
.../tcp/internal/TcpDiscoveryStatistics.java | 4 +
.../resources/META-INF/classnames.properties | 86 ++-
.../AbstractAffinityFunctionSelfTest.java | 2 +-
.../jdbc/JdbcTypesDefaultTransformerTest.java | 283 +++++++++
.../IgniteComputeTopologyExceptionTest.java | 5 +-
.../binary/BinaryMarshallerSelfTest.java | 66 ++
.../GridDiscoveryManagerAliveCacheSelfTest.java | 2 +-
.../CacheSerializableTransactionsTest.java | 604 +++++++++++++++++-
.../cache/GridCacheMvccFlagsTest.java | 8 +-
.../cache/GridCacheMvccPartitionedSelfTest.java | 334 ++++++++--
.../processors/cache/GridCacheMvccSelfTest.java | 212 +++----
.../GridCachePartitionedAffinitySpreadTest.java | 7 +-
.../processors/cache/GridCacheTestEntryEx.java | 77 +--
...heapCacheMetricsForClusterGroupSelfTest.java | 141 +++++
.../cache/OffheapCacheOnClientsTest.java | 143 +++++
.../distributed/dht/GridCacheDhtTestUtils.java | 232 -------
.../GridCacheRebalancingSyncSelfTest.java | 2 +
.../CacheOffHeapAndSwapMetricsSelfTest.java | 621 -------------------
...LocalCacheOffHeapAndSwapMetricsSelfTest.java | 621 +++++++++++++++++++
.../closure/GridClosureSerializationTest.java | 177 ++++++
...gniteServiceProxyTimeoutInitializedTest.java | 284 +++++++++
.../loadtests/hashmap/GridHashMapLoadTest.java | 7 +-
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
.../IgniteCacheMetricsSelfTestSuite.java | 6 +-
.../ignite/testsuites/IgniteCacheTestSuite.java | 2 +
.../testsuites/IgniteCacheTestSuite2.java | 2 +
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
.../resources/META-INF/classnames.properties | 114 ++++
.../processors/query/h2/IgniteH2Indexing.java | 77 ++-
.../h2/twostep/GridReduceQueryExecutor.java | 14 +-
...niteCachePartitionedFieldsQuerySelfTest.java | 25 +
101 files changed, 4783 insertions(+), 2473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
----------------------------------------------------------------------
diff --cc modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index 9058837,0000000..b4bed0d
mode 100644,000000..100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@@ -1,519 -1,0 +1,522 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.cassandra;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.Row;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import javax.cache.Cache;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cache.store.CacheStoreSession;
+import org.apache.ignite.cache.store.cassandra.datasource.DataSource;
+import org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings;
+import org.apache.ignite.cache.store.cassandra.persistence.PersistenceController;
+import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
+import org.apache.ignite.cache.store.cassandra.session.ExecutionAssistant;
+import org.apache.ignite.cache.store.cassandra.session.GenericBatchExecutionAssistant;
+import org.apache.ignite.cache.store.cassandra.session.LoadCacheCustomQueryWorker;
- import org.apache.ignite.cache.store.cassandra.session.transaction.DeleteMutation;
- import org.apache.ignite.cache.store.cassandra.session.transaction.Mutation;
- import org.apache.ignite.cache.store.cassandra.session.transaction.WriteMutation;
++import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.logger.NullLogger;
+import org.apache.ignite.resources.CacheStoreSessionResource;
+import org.apache.ignite.resources.LoggerResource;
+
+/**
+ * Implementation of {@link CacheStore} backed by Cassandra database.
+ *
+ * @param <K> Ignite cache key type.
+ * @param <V> Ignite cache value type.
+ */
+public class CassandraCacheStore<K, V> implements CacheStore<K, V> {
+ /** Buffer to store mutations performed withing transaction. */
+ private static final String TRANSACTION_BUFFER = "CASSANDRA_TRANSACTION_BUFFER";
+
+ /** Auto-injected store session. */
+ @SuppressWarnings("unused")
+ @CacheStoreSessionResource
+ private CacheStoreSession storeSes;
+
+ /** Auto-injected logger instance. */
+ @SuppressWarnings("unused")
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** Cassandra data source. */
+ private DataSource dataSrc;
+
+ /** Max workers thread count. These threads are responsible for load cache. */
+ private int maxPoolSize = Runtime.getRuntime().availableProcessors();
+
+ /** Controller component responsible for serialization logic. */
+ private final PersistenceController controller;
+
+ /**
+ * Store constructor.
+ *
+ * @param dataSrc Data source.
+ * @param settings Persistence settings for Ignite key and value objects.
+ * @param maxPoolSize Max workers thread count.
+ */
+ public CassandraCacheStore(DataSource dataSrc, KeyValuePersistenceSettings settings, int maxPoolSize) {
+ this.dataSrc = dataSrc;
+ this.controller = new PersistenceController(settings);
+ this.maxPoolSize = maxPoolSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void loadCache(IgniteBiInClosure<K, V> clo, Object... args) throws CacheLoaderException {
+ if (clo == null)
+ return;
+
+ if (args == null || args.length == 0)
+ args = new String[] {"select * from " + controller.getPersistenceSettings().getKeyspace() + "." + cassandraTable() + ";"};
+
+ ExecutorService pool = null;
+
+ Collection<Future<?>> futs = new ArrayList<>(args.length);
+
+ try {
+ pool = Executors.newFixedThreadPool(maxPoolSize);
+
+ CassandraSession ses = getCassandraSession();
+
+ for (Object obj : args) {
+ if (obj == null || !(obj instanceof String) || !((String)obj).trim().toLowerCase().startsWith("select"))
+ continue;
+
+ futs.add(pool.submit(new LoadCacheCustomQueryWorker<>(ses, (String) obj, controller, log, clo)));
+ }
+
+ for (Future<?> fut : futs)
+ U.get(fut);
+
+ if (log != null && log.isDebugEnabled() && storeSes != null)
+ log.debug("Cache loaded from db: " + storeSes.cacheName());
+ }
+ catch (IgniteCheckedException e) {
+ if (storeSes != null)
+ throw new CacheLoaderException("Failed to load Ignite cache: " + storeSes.cacheName(), e.getCause());
+ else
+ throw new CacheLoaderException("Failed to load cache", e.getCause());
+ }
+ finally {
+ U.shutdownNow(getClass(), pool, log);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sessionEnd(boolean commit) throws CacheWriterException {
+ if (!storeSes.isWithinTransaction())
+ return;
+
+ List<Mutation> mutations = mutations();
+ if (mutations == null || mutations.isEmpty())
+ return;
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ ses.execute(mutations);
+ }
+ finally {
+ mutations.clear();
+ U.closeQuiet(ses);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings({"unchecked"})
+ @Override public V load(final K key) throws CacheLoaderException {
+ if (key == null)
+ return null;
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ return ses.execute(new ExecutionAssistant<V>() {
+ /** {@inheritDoc} */
+ @Override public boolean tableExistenceRequired() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getTable() {
+ return cassandraTable();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatement() {
+ return controller.getLoadStatement(cassandraTable(), false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public BoundStatement bindStatement(PreparedStatement statement) {
+ return controller.bindKey(statement, key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyValuePersistenceSettings getPersistenceSettings() {
+ return controller.getPersistenceSettings();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String operationName() {
+ return "READ";
+ }
+
+ /** {@inheritDoc} */
+ @Override public V process(Row row) {
+ return row == null ? null : (V)controller.buildValueObject(row);
+ }
+ });
+ }
+ finally {
+ U.closeQuiet(ses);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public Map<K, V> loadAll(Iterable<? extends K> keys) throws CacheLoaderException {
+ if (keys == null || !keys.iterator().hasNext())
+ return new HashMap<>();
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ return ses.execute(new GenericBatchExecutionAssistant<Map<K, V>, K>() {
+ private Map<K, V> data = new HashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public String getTable() {
+ return cassandraTable();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatement() {
+ return controller.getLoadStatement(cassandraTable(), true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public BoundStatement bindStatement(PreparedStatement statement, K key) {
+ return controller.bindKey(statement, key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyValuePersistenceSettings getPersistenceSettings() {
+ return controller.getPersistenceSettings();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String operationName() {
+ return "BULK_READ";
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<K, V> processedData() {
+ return data;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void process(Row row) {
+ data.put((K)controller.buildKeyObject(row), (V)controller.buildValueObject(row));
+ }
+ }, keys);
+ }
+ finally {
+ U.closeQuiet(ses);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(final Cache.Entry<? extends K, ? extends V> entry) throws CacheWriterException {
+ if (entry == null || entry.getKey() == null)
+ return;
+
+ if (storeSes.isWithinTransaction()) {
+ accumulate(new WriteMutation(entry, cassandraTable(), controller));
+ return;
+ }
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ ses.execute(new ExecutionAssistant<Void>() {
+ /** {@inheritDoc} */
+ @Override public boolean tableExistenceRequired() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getTable() {
+ return cassandraTable();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatement() {
+ return controller.getWriteStatement(cassandraTable());
+ }
+
+ /** {@inheritDoc} */
+ @Override public BoundStatement bindStatement(PreparedStatement statement) {
+ return controller.bindKeyValue(statement, entry.getKey(), entry.getValue());
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyValuePersistenceSettings getPersistenceSettings() {
+ return controller.getPersistenceSettings();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String operationName() {
+ return "WRITE";
+ }
+
+ /** {@inheritDoc} */
+ @Override public Void process(Row row) {
+ return null;
+ }
+ });
+ }
+ finally {
+ U.closeQuiet(ses);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeAll(Collection<Cache.Entry<? extends K, ? extends V>> entries) throws CacheWriterException {
+ if (entries == null || entries.isEmpty())
+ return;
+
+ if (storeSes.isWithinTransaction()) {
+ for (Cache.Entry<?, ?> entry : entries)
+ accumulate(new WriteMutation(entry, cassandraTable(), controller));
+
+ return;
+ }
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ ses.execute(new GenericBatchExecutionAssistant<Void, Cache.Entry<? extends K, ? extends V>>() {
+ /** {@inheritDoc} */
+ @Override public String getTable() {
+ return cassandraTable();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatement() {
+ return controller.getWriteStatement(cassandraTable());
+ }
+
+ /** {@inheritDoc} */
+ @Override public BoundStatement bindStatement(PreparedStatement statement,
+ Cache.Entry<? extends K, ? extends V> entry) {
+ return controller.bindKeyValue(statement, entry.getKey(), entry.getValue());
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyValuePersistenceSettings getPersistenceSettings() {
+ return controller.getPersistenceSettings();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String operationName() {
+ return "BULK_WRITE";
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean tableExistenceRequired() {
+ return true;
+ }
+ }, entries);
+ }
+ finally {
+ U.closeQuiet(ses);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void delete(final Object key) throws CacheWriterException {
+ if (key == null)
+ return;
+
+ if (storeSes.isWithinTransaction()) {
+ accumulate(new DeleteMutation(key, cassandraTable(), controller));
+ return;
+ }
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ ses.execute(new ExecutionAssistant<Void>() {
+ /** {@inheritDoc} */
+ @Override public boolean tableExistenceRequired() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getTable() {
+ return cassandraTable();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatement() {
+ return controller.getDeleteStatement(cassandraTable());
+ }
+
+ /** {@inheritDoc} */
+ @Override public BoundStatement bindStatement(PreparedStatement statement) {
+ return controller.bindKey(statement, key);
+ }
+
+
+ /** {@inheritDoc} */
+ @Override public KeyValuePersistenceSettings getPersistenceSettings() {
+ return controller.getPersistenceSettings();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String operationName() {
+ return "DELETE";
+ }
+
+ /** {@inheritDoc} */
+ @Override public Void process(Row row) {
+ return null;
+ }
+ });
+ }
+ finally {
+ U.closeQuiet(ses);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void deleteAll(Collection<?> keys) throws CacheWriterException {
+ if (keys == null || keys.isEmpty())
+ return;
+
+ if (storeSes.isWithinTransaction()) {
+ for (Object key : keys)
+ accumulate(new DeleteMutation(key, cassandraTable(), controller));
+
+ return;
+ }
+
+ CassandraSession ses = getCassandraSession();
+
+ try {
+ ses.execute(new GenericBatchExecutionAssistant<Void, Object>() {
+ /** {@inheritDoc} */
+ @Override public String getTable() {
+ return cassandraTable();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStatement() {
+ return controller.getDeleteStatement(cassandraTable());
+ }
+
+ /** {@inheritDoc} */
+ @Override public BoundStatement bindStatement(PreparedStatement statement, Object key) {
+ return controller.bindKey(statement, key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyValuePersistenceSettings getPersistenceSettings() {
+ return controller.getPersistenceSettings();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String operationName() {
+ return "BULK_DELETE";
+ }
+ }, keys);
+ }
+ finally {
+ U.closeQuiet(ses);
+ }
+ }
+
+ /**
+ * Gets Cassandra session wrapper or creates new if it doesn't exist.
+ * This wrapper hides all the low-level Cassandra interaction details by providing only high-level methods.
+ *
+ * @return Cassandra session wrapper.
+ */
+ private CassandraSession getCassandraSession() {
+ return dataSrc.session(log != null ? log : new NullLogger());
+ }
+
+ /**
+ * Returns table name to use for all Cassandra based operations (READ/WRITE/DELETE).
+ *
+ * @return Table name.
+ */
+ private String cassandraTable() {
+ return controller.getPersistenceSettings().getTable() != null ?
+ controller.getPersistenceSettings().getTable() : storeSes.cacheName().trim().toLowerCase();
+ }
+
+ /**
+ * Accumulates mutation in the transaction buffer.
+ *
+ * @param mutation Mutation operation.
+ */
+ private void accumulate(Mutation mutation) {
+ //noinspection unchecked
+ List<Mutation> mutations = (List<Mutation>)storeSes.properties().get(TRANSACTION_BUFFER);
+
+ if (mutations == null) {
+ mutations = new LinkedList<>();
+ storeSes.properties().put(TRANSACTION_BUFFER, mutations);
+ }
+
+ mutations.add(mutation);
+ }
+
+ /**
+ * Returns all the mutations performed withing transaction.
+ *
+ * @return Mutations
+ */
+ private List<Mutation> mutations() {
+ //noinspection unchecked
+ return (List<Mutation>)storeSes.properties().get(TRANSACTION_BUFFER);
+ }
++
++ /** {@inheritDoc} */
++ @Override public String toString() {
++ return S.toString(CassandraCacheStore.class, this);
++ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
----------------------------------------------------------------------
diff --cc modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index f582aac,0000000..1ba3c7d
mode 100644,000000..100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@@ -1,647 -1,0 +1,656 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.cassandra.datasource;
+
+import com.datastax.driver.core.AuthProvider;
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.ConsistencyLevel;
+import com.datastax.driver.core.NettyOptions;
+import com.datastax.driver.core.PoolingOptions;
+import com.datastax.driver.core.ProtocolOptions;
+import com.datastax.driver.core.ProtocolVersion;
+import com.datastax.driver.core.SSLOptions;
+import com.datastax.driver.core.SocketOptions;
+import com.datastax.driver.core.policies.AddressTranslator;
+import com.datastax.driver.core.policies.LoadBalancingPolicy;
+import com.datastax.driver.core.policies.ReconnectionPolicy;
+import com.datastax.driver.core.policies.RetryPolicy;
+import com.datastax.driver.core.policies.SpeculativeExecutionPolicy;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
+import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
+import org.apache.ignite.internal.util.typedef.internal.U;
++import org.apache.ignite.internal.util.tostring.GridToStringExclude;
++import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Data source abstraction to specify configuration of the Cassandra session to be used.
+ */
+public class DataSource implements Externalizable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Null object, used as a replacement for those Cassandra connection options which
+ * don't support serialization (RetryPolicy, LoadBalancingPolicy and etc).
+ */
+ private static final UUID NULL_OBJECT = UUID.fromString("45ffae47-3193-5910-84a2-048fe65735d9");
+
+ /** Number of rows to immediately fetch in CQL statement execution. */
+ private Integer fetchSize;
+
+ /** Consistency level for READ operations. */
+ private ConsistencyLevel readConsistency;
+
+ /** Consistency level for WRITE operations. */
+ private ConsistencyLevel writeConsistency;
+
+ /** Username to use for authentication. */
++ @GridToStringExclude
+ private String user;
+
+ /** Password to use for authentication. */
++ @GridToStringExclude
+ private String pwd;
+
+ /** Port to use for Cassandra connection. */
+ private Integer port;
+
+ /** List of contact points to connect to Cassandra cluster. */
+ private List<InetAddress> contactPoints;
+
+ /** List of contact points with ports to connect to Cassandra cluster. */
+ private List<InetSocketAddress> contactPointsWithPorts;
+
+ /** Maximum time to wait for schema agreement before returning from a DDL query. */
+ private Integer maxSchemaAgreementWaitSeconds;
+
+ /** The native protocol version to use. */
+ private Integer protoVer;
+
+ /** Compression to use for the transport. */
+ private String compression;
+
+ /** Use SSL for communications with Cassandra. */
+ private Boolean useSSL;
+
+ /** Enables metrics collection. */
+ private Boolean collectMetrix;
+
+ /** Enables JMX reporting of the metrics. */
+ private Boolean jmxReporting;
+
+ /** Credentials to use for authentication. */
+ private Credentials creds;
+
+ /** Load balancing policy to use. */
+ private LoadBalancingPolicy loadBalancingPlc;
+
+ /** Reconnection policy to use. */
+ private ReconnectionPolicy reconnectionPlc;
+
+ /** Retry policy to use. */
+ private RetryPolicy retryPlc;
+
+ /** Address translator to use. */
+ private AddressTranslator addrTranslator;
+
+ /** Speculative execution policy to use. */
+ private SpeculativeExecutionPolicy speculativeExecutionPlc;
+
+ /** Authentication provider to use. */
+ private AuthProvider authProvider;
+
+ /** SSL options to use. */
+ private SSLOptions sslOptions;
+
+ /** Connection pooling options to use. */
+ private PoolingOptions poolingOptions;
+
+ /** Socket options to use. */
+ private SocketOptions sockOptions;
+
+ /** Netty options to use for connection. */
+ private NettyOptions nettyOptions;
+
+ /** Cassandra session wrapper instance. */
+ private volatile CassandraSession ses;
+
+ /**
+ * Sets user name to use for authentication.
+ *
+ * @param user user name
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setUser(String user) {
+ this.user = user;
+
+ invalidate();
+ }
+
+ /**
+ * Sets password to use for authentication.
+ *
+ * @param pwd password
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setPassword(String pwd) {
+ this.pwd = pwd;
+
+ invalidate();
+ }
+
+ /**
+ * Sets port to use for Cassandra connection.
+ *
+ * @param port port
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setPort(int port) {
+ this.port = port;
+
+ invalidate();
+ }
+
+ /**
+ * Sets list of contact points to connect to Cassandra cluster.
+ *
+ * @param points contact points
+ */
+ public void setContactPoints(String... points) {
+ if (points == null || points.length == 0)
+ return;
+
+ for (String point : points) {
+ if (point.contains(":")) {
+ if (contactPointsWithPorts == null)
+ contactPointsWithPorts = new LinkedList<>();
+
+ String[] chunks = point.split(":");
+
+ try {
+ contactPointsWithPorts.add(InetSocketAddress.createUnresolved(chunks[0].trim(), Integer.parseInt(chunks[1].trim())));
+ }
+ catch (Throwable e) {
+ throw new IllegalArgumentException("Incorrect contact point '" + point + "' specified for Cassandra cache storage", e);
+ }
+ }
+ else {
+ if (contactPoints == null)
+ contactPoints = new LinkedList<>();
+
+ try {
+ contactPoints.add(InetAddress.getByName(point));
+ }
+ catch (Throwable e) {
+ throw new IllegalArgumentException("Incorrect contact point '" + point + "' specified for Cassandra cache storage", e);
+ }
+ }
+ }
+
+ invalidate();
+ }
+
+ /** Sets maximum time to wait for schema agreement before returning from a DDL query. */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setMaxSchemaAgreementWaitSeconds(int seconds) {
+ maxSchemaAgreementWaitSeconds = seconds;
+
+ invalidate();
+ }
+
+ /**
+ * Sets the native protocol version to use.
+ *
+ * @param ver version number
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setProtocolVersion(int ver) {
+ protoVer = ver;
+
+ invalidate();
+ }
+
+ /**
+ * Sets compression algorithm to use for the transport.
+ *
+ * @param compression Compression algorithm.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setCompression(String compression) {
+ this.compression = compression == null || compression.trim().isEmpty() ? null : compression.trim();
+
+ try {
+ if (this.compression != null)
+ ProtocolOptions.Compression.valueOf(this.compression);
+ }
+ catch (Throwable e) {
+ throw new IgniteException("Incorrect compression '" + compression + "' specified for Cassandra connection", e);
+ }
+
+ invalidate();
+ }
+
+ /**
+ * Enables SSL for communications with Cassandra.
+ *
+ * @param use Flag to enable/disable SSL.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setUseSSL(boolean use) {
+ useSSL = use;
+
+ invalidate();
+ }
+
+ /**
+ * Enables metrics collection.
+ *
+ * @param collect Flag to enable/disable metrics collection.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setCollectMetrix(boolean collect) {
+ collectMetrix = collect;
+
+ invalidate();
+ }
+
+ /**
+ * Enables JMX reporting of the metrics.
+ *
+ * @param enableReporting Flag to enable/disable JMX reporting.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setJmxReporting(boolean enableReporting) {
+ jmxReporting = enableReporting;
+
+ invalidate();
+ }
+
+ /**
+ * Sets number of rows to immediately fetch in CQL statement execution.
+ *
+ * @param size Number of rows to fetch.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setFetchSize(int size) {
+ fetchSize = size;
+
+ invalidate();
+ }
+
+ /**
+ * Set consistency level for READ operations.
+ *
+ * @param level Consistency level.
+ */
+ public void setReadConsistency(String level) {
+ readConsistency = parseConsistencyLevel(level);
+
+ invalidate();
+ }
+
+ /**
+ * Set consistency level for WRITE operations.
+ *
+ * @param level Consistency level.
+ */
+ public void setWriteConsistency(String level) {
+ writeConsistency = parseConsistencyLevel(level);
+
+ invalidate();
+ }
+
+ /**
+ * Sets credentials to use for authentication.
+ *
+ * @param creds Credentials.
+ */
+ public void setCredentials(Credentials creds) {
+ this.creds = creds;
+
+ invalidate();
+ }
+
+ /**
+ * Sets load balancing policy.
+ *
+ * @param plc Load balancing policy.
+ */
+ public void setLoadBalancingPolicy(LoadBalancingPolicy plc) {
+ loadBalancingPlc = plc;
+
+ invalidate();
+ }
+
+ /**
+ * Sets reconnection policy.
+ *
+ * @param plc Reconnection policy.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setReconnectionPolicy(ReconnectionPolicy plc) {
+ reconnectionPlc = plc;
+
+ invalidate();
+ }
+
+ /**
+ * Sets retry policy.
+ *
+ * @param plc Retry policy.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setRetryPolicy(RetryPolicy plc) {
+ retryPlc = plc;
+
+ invalidate();
+ }
+
+ /**
+ * Sets address translator.
+ *
+ * @param translator Address translator.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setAddressTranslator(AddressTranslator translator) {
+ addrTranslator = translator;
+
+ invalidate();
+ }
+
+ /**
+ * Sets speculative execution policy.
+ *
+ * @param plc Speculative execution policy.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setSpeculativeExecutionPolicy(SpeculativeExecutionPolicy plc) {
+ speculativeExecutionPlc = plc;
+
+ invalidate();
+ }
+
+ /**
+ * Sets authentication provider.
+ *
+ * @param provider Authentication provider.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setAuthProvider(AuthProvider provider) {
+ authProvider = provider;
+
+ invalidate();
+ }
+
+ /**
+ * Sets SSL options.
+ *
+ * @param options SSL options.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setSslOptions(SSLOptions options) {
+ sslOptions = options;
+
+ invalidate();
+ }
+
+ /**
+ * Sets pooling options.
+ *
+ * @param options pooling options to use.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setPoolingOptions(PoolingOptions options) {
+ poolingOptions = options;
+
+ invalidate();
+ }
+
+ /**
+ * Sets socket options to use.
+ *
+ * @param options Socket options.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setSocketOptions(SocketOptions options) {
+ sockOptions = options;
+
+ invalidate();
+ }
+
+ /**
+ * Sets netty options to use.
+ *
+ * @param options netty options.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void setNettyOptions(NettyOptions options) {
+ nettyOptions = options;
+
+ invalidate();
+ }
+
+ /**
+ * Creates Cassandra session wrapper if it wasn't created yet and returns it
+ *
+ * @param log logger
+ * @return Cassandra session wrapper
+ */
+ @SuppressWarnings("deprecation")
+ public synchronized CassandraSession session(IgniteLogger log) {
+ if (ses != null)
+ return ses;
+
+ Cluster.Builder builder = Cluster.builder();
+
+ if (user != null)
+ builder = builder.withCredentials(user, pwd);
+
+ if (port != null)
+ builder = builder.withPort(port);
+
+ if (contactPoints != null)
+ builder = builder.addContactPoints(contactPoints);
+
+ if (contactPointsWithPorts != null)
+ builder = builder.addContactPointsWithPorts(contactPointsWithPorts);
+
+ if (maxSchemaAgreementWaitSeconds != null)
+ builder = builder.withMaxSchemaAgreementWaitSeconds(maxSchemaAgreementWaitSeconds);
+
+ if (protoVer != null)
+ builder = builder.withProtocolVersion(ProtocolVersion.fromInt(protoVer));
+
+ if (compression != null) {
+ try {
+ builder = builder.withCompression(ProtocolOptions.Compression.valueOf(compression.trim().toLowerCase()));
+ }
+ catch (IllegalArgumentException e) {
+ throw new IgniteException("Incorrect compression option '" + compression + "' specified for Cassandra connection", e);
+ }
+ }
+
+ if (useSSL != null && useSSL)
+ builder = builder.withSSL();
+
+ if (sslOptions != null)
+ builder = builder.withSSL(sslOptions);
+
+ if (collectMetrix != null && !collectMetrix)
+ builder = builder.withoutMetrics();
+
+ if (jmxReporting != null && !jmxReporting)
+ builder = builder.withoutJMXReporting();
+
+ if (creds != null)
+ builder = builder.withCredentials(creds.getUser(), creds.getPassword());
+
+ if (loadBalancingPlc != null)
+ builder = builder.withLoadBalancingPolicy(loadBalancingPlc);
+
+ if (reconnectionPlc != null)
+ builder = builder.withReconnectionPolicy(reconnectionPlc);
+
+ if (retryPlc != null)
+ builder = builder.withRetryPolicy(retryPlc);
+
+ if (addrTranslator != null)
+ builder = builder.withAddressTranslator(addrTranslator);
+
+ if (speculativeExecutionPlc != null)
+ builder = builder.withSpeculativeExecutionPolicy(speculativeExecutionPlc);
+
+ if (authProvider != null)
+ builder = builder.withAuthProvider(authProvider);
+
+ if (poolingOptions != null)
+ builder = builder.withPoolingOptions(poolingOptions);
+
+ if (sockOptions != null)
+ builder = builder.withSocketOptions(sockOptions);
+
+ if (nettyOptions != null)
+ builder = builder.withNettyOptions(nettyOptions);
+
+ return ses = new CassandraSessionImpl(builder, fetchSize, readConsistency, writeConsistency, log);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(fetchSize);
+ out.writeObject(readConsistency);
+ out.writeObject(writeConsistency);
+ U.writeString(out, user);
+ U.writeString(out, pwd);
+ out.writeObject(port);
+ out.writeObject(contactPoints);
+ out.writeObject(contactPointsWithPorts);
+ out.writeObject(maxSchemaAgreementWaitSeconds);
+ out.writeObject(protoVer);
+ U.writeString(out, compression);
+ out.writeObject(useSSL);
+ out.writeObject(collectMetrix);
+ out.writeObject(jmxReporting);
+ out.writeObject(creds);
+ writeObject(out, loadBalancingPlc);
+ writeObject(out, reconnectionPlc);
+ writeObject(out, addrTranslator);
+ writeObject(out, speculativeExecutionPlc);
+ writeObject(out, authProvider);
+ writeObject(out, sslOptions);
+ writeObject(out, poolingOptions);
+ writeObject(out, sockOptions);
+ writeObject(out, nettyOptions);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ fetchSize = (Integer)in.readObject();
+ readConsistency = (ConsistencyLevel)in.readObject();
+ writeConsistency = (ConsistencyLevel)in.readObject();
+ user = U.readString(in);
+ pwd = U.readString(in);
+ port = (Integer)in.readObject();
+ contactPoints = (List<InetAddress>)in.readObject();
+ contactPointsWithPorts = (List<InetSocketAddress>)in.readObject();
+ maxSchemaAgreementWaitSeconds = (Integer)in.readObject();
+ protoVer = (Integer)in.readObject();
+ compression = U.readString(in);
+ useSSL = (Boolean)in.readObject();
+ collectMetrix = (Boolean)in.readObject();
+ jmxReporting = (Boolean)in.readObject();
+ creds = (Credentials)in.readObject();
+ loadBalancingPlc = (LoadBalancingPolicy)readObject(in);
+ reconnectionPlc = (ReconnectionPolicy)readObject(in);
+ addrTranslator = (AddressTranslator)readObject(in);
+ speculativeExecutionPlc = (SpeculativeExecutionPolicy)readObject(in);
+ authProvider = (AuthProvider)readObject(in);
+ sslOptions = (SSLOptions)readObject(in);
+ poolingOptions = (PoolingOptions)readObject(in);
+ sockOptions = (SocketOptions)readObject(in);
+ nettyOptions = (NettyOptions)readObject(in);
+ }
+
+ /**
+ * Helper method used to serialize class members
+ * @param out the stream to write the object to
+ * @param obj the object to be written
+ * @throws IOException Includes any I/O exceptions that may occur
+ */
+ private void writeObject(ObjectOutput out, Object obj) throws IOException {
+ out.writeObject(obj == null || !(obj instanceof Serializable) ? NULL_OBJECT : obj);
+ }
+
+ /**
+ * Helper method used to deserialize class members
+ * @param in the stream to read data from in order to restore the object
+ * @throws IOException Includes any I/O exceptions that may occur
+ * @throws ClassNotFoundException If the class for an object being restored cannot be found
+ * @return deserialized object
+ */
+ private Object readObject(ObjectInput in) throws IOException, ClassNotFoundException {
+ Object obj = in.readObject();
+ return NULL_OBJECT.equals(obj) ? null : obj;
+ }
+
+ /**
+ * Parses consistency level provided as string.
+ *
+ * @param level consistency level string.
+ *
+ * @return consistency level.
+ */
+ private ConsistencyLevel parseConsistencyLevel(String level) {
+ if (level == null)
+ return null;
+
+ try {
+ return ConsistencyLevel.valueOf(level.trim().toUpperCase());
+ }
+ catch (Throwable e) {
+ throw new IgniteException("Incorrect consistency level '" + level + "' specified for Cassandra connection", e);
+ }
+ }
+
+ /**
+ * Invalidates session.
+ */
+ private synchronized void invalidate() {
+ ses = null;
+ }
++
++ /** {@inheritDoc} */
++ @Override public String toString() {
++ return S.toString(DataSource.class, this);
++ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 5f0b8a0,4d59d50..8187e8f
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@@ -3076,8 -3233,8 +3076,8 @@@ public abstract class GridCacheAdapter<
}
/** {@inheritDoc} */
- @Override public CacheMetrics clusterMetrics() {
+ @Override public final CacheMetrics clusterMetrics() {
- return clusterMetrics(ctx.grid().cluster().forCacheNodes(ctx.name()));
+ return clusterMetrics(ctx.grid().cluster().forDataNodes(ctx.name()));
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index aeb3ef4,3690f35..d26242d
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@@ -1587,9 -1590,9 +1590,9 @@@ public class GridServiceProcessor exten
else
topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
- depExe.submit(new BusyRunnable() {
+ depExe.execute(new BusyRunnable() {
@Override public void run0() {
- ClusterNode oldest = CU.oldestAliveCacheServerNode(cache.context().shared(), topVer);
+ ClusterNode oldest = ctx.discovery().oldestAliveCacheServerNode(topVer);
if (oldest != null && oldest.isLocal()) {
final Collection<GridServiceDeployment> retries = new ConcurrentLinkedQueue<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
index ee5b65c,43017db..8f8d78a
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AbstractAffinityFunctionSelfTest.java
@@@ -104,24 -104,9 +104,25 @@@ public abstract class AbstractAffinityF
}
/**
+ * @param backups Number of backups.
* @throws Exception If failed.
*/
+ public void testNullKeyForPartitionCalculation() throws Exception {
+ AffinityFunction aff = affinityFunction();
+
+ try {
+ aff.partition(null);
+
+ fail("Should throw IllegalArgumentException due to NULL affinity key.");
+ } catch (IllegalArgumentException e) {
+ e.getMessage().contains("Null key is passed for a partition calculation. " +
+ "Make sure that an affinity key that is used is initialized properly.");
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
protected void checkNodeRemoved(int backups) throws Exception {
checkNodeRemoved(backups, 1, 1);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 554bb3d,deec72a..1e73e79
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@@ -39,10 -39,7 +39,11 @@@ import org.apache.ignite.cache.store.jd
import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
+import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
+import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
+ import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 2d06f3a,350b715..b28619c
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@@ -66,7 -66,7 +66,8 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingDefaultMarshallerTest;
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingJdkMarshallerTest;
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest;
+import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesSelfTest;
+ import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
@@@ -142,7 -142,7 +143,8 @@@ public class IgniteKernalSelfTestSuite
suite.addTestSuite(GridServiceProxyNodeStopSelfTest.class);
suite.addTestSuite(GridServiceProxyClientReconnectSelfTest.class);
suite.addTestSuite(IgniteServiceReassignmentTest.class);
+ suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
+ suite.addTestSuite(IgniteServiceDynamicCachesSelfTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 5df44db,362ddd8..c541185
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@@ -794,40 -774,29 +794,45 @@@ public class IgniteH2Indexing implement
throws IgniteCheckedException {
final Connection conn = connectionForSpace(spaceName);
- initLocalQueryContext(conn, enforceJoinOrder, filters);
+ setupConnection(conn, false, enforceJoinOrder);
- Prepared p = null;
+ final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
- try {
- final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
++ Prepared p = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
+
- p = GridSqlQueryParser.prepared((JdbcPreparedStatement) stmt);
++ if (!p.isQuery()) {
++ GridH2QueryContext.clearThreadLocal();
+
- if (!p.isQuery()) {
- GridH2QueryContext.clearThreadLocal();
++ SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry);
+
- SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry);
++ if (params != null)
++ fldsQry.setArgs(params.toArray());
+
- if (params != null)
- fldsQry.setArgs(params.toArray());
++ fldsQry.setEnforceJoinOrder(enforceJoinOrder);
++ fldsQry.setTimeout(timeout, TimeUnit.MILLISECONDS);
+
- fldsQry.setEnforceJoinOrder(enforceJoinOrder);
- fldsQry.setTimeout(timeout, TimeUnit.MILLISECONDS);
++ return dmlProc.updateLocalSqlFields(spaceName, stmt, fldsQry, filters, cancel);
++ }
+
- return dmlProc.updateLocalSqlFields(spaceName, stmt, fldsQry, filters, cancel);
- }
+ List<GridQueryFieldMetadata> meta;
- List<GridQueryFieldMetadata> meta;
+ try {
+ meta = meta(stmt.getMetaData());
+ }
+ catch (SQLException e) {
+ throw new IgniteCheckedException("Cannot prepare query metadata", e);
+ }
- try {
- meta = meta(stmt.getMetaData());
- }
- catch (SQLException e) {
- throw new IgniteCheckedException("Cannot prepare query metadata", e);
- }
+ final GridH2QueryContext ctx = new GridH2QueryContext(nodeId, nodeId, 0, LOCAL)
+ .filter(filters).distributedJoins(false);
- return new GridQueryFieldsResultAdapter(meta, null) {
- @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException{
+ return new GridQueryFieldsResultAdapter(meta, null) {
+ @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException {
+ assert GridH2QueryContext.get() == null;
+
+ GridH2QueryContext.set(ctx);
+
+ try {
ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
return new FieldsIterator(rs);
http://git-wip-us.apache.org/repos/asf/ignite/blob/64247b92/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
[25/50] [abbrv] ignite git commit: IGNITE-4386: Hadoop: implemented
client cleanup on protocol close. This closes #1327. This closes #1339.
Posted by sb...@apache.org.
IGNITE-4386: Hadoop: implemented client cleanup on protocol close. This closes #1327. This closes #1339.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ffe53eb5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ffe53eb5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ffe53eb5
Branch: refs/heads/master
Commit: ffe53eb5a59908db3684ce11474cb875c4bf392d
Parents: a976c42
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 12 11:29:23 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:24 2016 +0300
----------------------------------------------------------------------
.../IgniteHadoopClientProtocolProvider.java | 70 ++----
.../hadoop/impl/proto/HadoopClientProtocol.java | 55 +++--
.../hadoop/mapreduce/MapReduceClient.java | 147 ++++++++++++
...opClientProtocolMultipleServersSelfTest.java | 93 +++-----
.../client/HadoopClientProtocolSelfTest.java | 228 ++++++++++---------
5 files changed, 367 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ffe53eb5/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
index 1efe625..920e8b7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopClientProtocolProvider.java
@@ -23,24 +23,16 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.ConcurrentHashMap;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
-import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.ConnectorConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.client.GridClient;
-import org.apache.ignite.internal.client.GridClientConfiguration;
-import org.apache.ignite.internal.client.GridClientException;
-import org.apache.ignite.internal.client.GridClientFactory;
-import org.apache.ignite.internal.client.marshaller.jdk.GridClientJdkMarshaller;
import org.apache.ignite.internal.processors.hadoop.impl.proto.HadoopClientProtocol;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.processors.hadoop.mapreduce.MapReduceClient;
import org.apache.ignite.internal.util.typedef.F;
-import static org.apache.ignite.internal.client.GridClientProtocol.TCP;
-
/**
* Ignite Hadoop client protocol provider.
@@ -50,7 +42,7 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
public static final String FRAMEWORK_NAME = "ignite";
/** Clients. */
- private static final ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, MapReduceClient> cliMap = new ConcurrentHashMap<>();
/** {@inheritDoc} */
@Override public ClientProtocol create(Configuration conf) throws IOException {
@@ -91,7 +83,12 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
/** {@inheritDoc} */
@Override public void close(ClientProtocol cliProto) throws IOException {
- // No-op.
+ if (cliProto instanceof HadoopClientProtocol) {
+ MapReduceClient cli = ((HadoopClientProtocol)cliProto).client();
+
+ if (cli.release())
+ cliMap.remove(cli.cluster(), cli);
+ }
}
/**
@@ -102,7 +99,7 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
* @return Client protocol.
* @throws IOException If failed.
*/
- private static ClientProtocol createProtocol(String addr, Configuration conf) throws IOException {
+ private ClientProtocol createProtocol(String addr, Configuration conf) throws IOException {
return new HadoopClientProtocol(conf, client(addr, Collections.singletonList(addr)));
}
@@ -114,45 +111,24 @@ public class IgniteHadoopClientProtocolProvider extends ClientProtocolProvider {
* @return Client.
* @throws IOException If failed.
*/
- private static GridClient client(String clusterName, Collection<String> addrs) throws IOException {
- try {
- IgniteInternalFuture<GridClient> fut = cliMap.get(clusterName);
-
- if (fut == null) {
- GridFutureAdapter<GridClient> fut0 = new GridFutureAdapter<>();
-
- IgniteInternalFuture<GridClient> oldFut = cliMap.putIfAbsent(clusterName, fut0);
+ @SuppressWarnings("unchecked")
+ private MapReduceClient client(String clusterName, Collection<String> addrs) throws IOException {
+ while (true) {
+ MapReduceClient cli = cliMap.get(clusterName);
- if (oldFut != null)
- return oldFut.get();
- else {
- GridClientConfiguration cliCfg = new GridClientConfiguration();
+ if (cli == null) {
+ cli = new MapReduceClient(clusterName, addrs);
- cliCfg.setProtocol(TCP);
- cliCfg.setServers(addrs);
- cliCfg.setMarshaller(new GridClientJdkMarshaller());
- cliCfg.setMaxConnectionIdleTime(24 * 60 * 60 * 1000L); // 1 day.
- cliCfg.setDaemon(true);
+ MapReduceClient oldCli = cliMap.putIfAbsent(clusterName, cli);
- try {
- GridClient cli = GridClientFactory.start(cliCfg);
-
- fut0.onDone(cli);
-
- return cli;
- }
- catch (GridClientException e) {
- fut0.onDone(e);
-
- throw new IOException("Failed to establish connection with Ignite: " + addrs, e);
- }
- }
+ if (oldCli != null)
+ cli = oldCli;
}
+
+ if (cli.acquire())
+ return cli;
else
- return fut.get();
- }
- catch (IgniteCheckedException e) {
- throw new IOException("Failed to establish connection with Ignite \u0441\u0434\u0433\u044b\u0435: " + addrs, e);
+ cliMap.remove(clusterName, cli);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ffe53eb5/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
index be2aa09..7fc0e77 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/proto/HadoopClientProtocol.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.processors.hadoop.mapreduce.MapReduceClient;
import org.apache.ignite.internal.client.GridClientException;
import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
@@ -78,7 +78,7 @@ public class HadoopClientProtocol implements ClientProtocol {
private final Configuration conf;
/** Ignite client. */
- private volatile GridClient cli;
+ private final MapReduceClient cli;
/** Last received version. */
private long lastVer = -1;
@@ -90,9 +90,10 @@ public class HadoopClientProtocol implements ClientProtocol {
* Constructor.
*
* @param conf Configuration.
- * @param cli Ignite client.
+ * @param cli Client.
*/
- public HadoopClientProtocol(Configuration conf, GridClient cli) {
+ public HadoopClientProtocol(Configuration conf, MapReduceClient cli) {
+ assert conf != null;
assert cli != null;
this.conf = conf;
@@ -104,7 +105,7 @@ public class HadoopClientProtocol implements ClientProtocol {
try {
conf.setLong(HadoopCommonUtils.REQ_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
- HadoopJobId jobID = cli.compute().execute(HadoopProtocolNextTaskIdTask.class.getName(), null);
+ HadoopJobId jobID = execute(HadoopProtocolNextTaskIdTask.class);
conf.setLong(HadoopCommonUtils.RESPONSE_NEW_JOBID_TS_PROPERTY, U.currentTimeMillis());
@@ -121,8 +122,8 @@ public class HadoopClientProtocol implements ClientProtocol {
try {
conf.setLong(HadoopCommonUtils.JOB_SUBMISSION_START_TS_PROPERTY, U.currentTimeMillis());
- HadoopJobStatus status = cli.compute().execute(HadoopProtocolSubmitJobTask.class.getName(),
- new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf)));
+ HadoopJobStatus status = execute(HadoopProtocolSubmitJobTask.class,
+ jobId.getJtIdentifier(), jobId.getId(), createJobInfo(conf));
if (status == null)
throw new IOException("Failed to submit job (null status obtained): " + jobId);
@@ -157,8 +158,7 @@ public class HadoopClientProtocol implements ClientProtocol {
/** {@inheritDoc} */
@Override public void killJob(JobID jobId) throws IOException, InterruptedException {
try {
- cli.compute().execute(HadoopProtocolKillJobTask.class.getName(),
- new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
+ execute(HadoopProtocolKillJobTask.class, jobId.getJtIdentifier(), jobId.getId());
}
catch (GridClientException e) {
throw new IOException("Failed to kill job: " + jobId, e);
@@ -181,11 +181,12 @@ public class HadoopClientProtocol implements ClientProtocol {
try {
Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1);
- HadoopProtocolTaskArguments args = delay >= 0 ?
- new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId(), delay) :
- new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId());
+ HadoopJobStatus status;
- HadoopJobStatus status = cli.compute().execute(HadoopProtocolJobStatusTask.class.getName(), args);
+ if (delay >= 0)
+ status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId(), delay);
+ else
+ status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId());
if (status == null)
throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
@@ -200,8 +201,8 @@ public class HadoopClientProtocol implements ClientProtocol {
/** {@inheritDoc} */
@Override public Counters getJobCounters(JobID jobId) throws IOException, InterruptedException {
try {
- final HadoopCounters counters = cli.compute().execute(HadoopProtocolJobCountersTask.class.getName(),
- new HadoopProtocolTaskArguments(jobId.getJtIdentifier(), jobId.getId()));
+ final HadoopCounters counters = execute(HadoopProtocolJobCountersTask.class,
+ jobId.getJtIdentifier(), jobId.getId());
if (counters == null)
throw new IOException("Job tracker doesn't have any information about the job: " + jobId);
@@ -329,6 +330,21 @@ public class HadoopClientProtocol implements ClientProtocol {
}
/**
+ * Execute task.
+ *
+ * @param taskCls Task class.
+ * @param args Arguments.
+ * @return Result.
+ * @throws IOException If failed.
+ * @throws GridClientException If failed.
+ */
+ private <T> T execute(Class taskCls, Object... args) throws IOException, GridClientException {
+ HadoopProtocolTaskArguments args0 = args != null ? new HadoopProtocolTaskArguments(args) : null;
+
+ return cli.client().compute().execute(taskCls.getName(), args0);
+ }
+
+ /**
* Process received status update.
*
* @param status Ignite status.
@@ -351,4 +367,13 @@ public class HadoopClientProtocol implements ClientProtocol {
return HadoopUtils.status(lastStatus, conf);
}
+
+ /**
+ * Gets the GridClient data.
+ *
+ * @return The client data.
+ */
+ public MapReduceClient client() {
+ return cli;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/ffe53eb5/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java
new file mode 100644
index 0000000..3d52176
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/mapreduce/MapReduceClient.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.mapreduce;
+
+import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientException;
+import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.client.marshaller.jdk.GridClientJdkMarshaller;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.ignite.internal.client.GridClientProtocol.TCP;
+
+/**
+ * Client.
+ */
+public class MapReduceClient {
+ /** Cluster name. */
+ private final String cluster;
+
+ /** Addresses. */
+ private final Collection<String> addrs;
+
+ /** Mutex. */
+ private final Object mux = new Object();
+
+ /** Usage counter. */
+ private final AtomicInteger cnt = new AtomicInteger();
+
+ /** Client. */
+ private volatile GridClient cli;
+
+ /**
+ * Constructor.
+ *
+ * @param cluster Cluster name.
+ * @param addrs Addresses.
+ */
+ public MapReduceClient(String cluster, Collection<String> addrs) {
+ this.cluster = cluster;
+ this.addrs = addrs;
+ }
+
+ /**
+ * @return Cluster name..
+ */
+ public String cluster() {
+ return cluster;
+ }
+
+ /**
+ * Gets the client.
+ *
+ * @return The client.
+ */
+ public GridClient client() throws IOException {
+ GridClient cli0 = cli;
+
+ if (cli0 == null) {
+ synchronized (mux) {
+ cli0 = cli;
+
+ if (cli0 == null) {
+ GridClientConfiguration cliCfg = new GridClientConfiguration();
+
+ cliCfg.setProtocol(TCP);
+ cliCfg.setServers(addrs);
+ cliCfg.setMarshaller(new GridClientJdkMarshaller());
+ cliCfg.setMaxConnectionIdleTime(24 * 60 * 60 * 1000L); // 1 day.
+ cliCfg.setDaemon(true);
+
+ try {
+ cli0 = GridClientFactory.start(cliCfg);
+
+ cli = cli0;
+ }
+ catch (GridClientException e) {
+ throw new IOException("Failed to establish connection with Ignite: " + addrs, e);
+ }
+ }
+ }
+ }
+
+ return cli0;
+ }
+
+ /**
+ * Increments usage count.
+ *
+ * @return {@code True} if succeeded and client can be used.
+ */
+ public boolean acquire() {
+ while (true) {
+ int cur = cnt.get();
+
+ if (cur < 0)
+ return false;
+
+ int next = cur + 1;
+
+ if (cnt.compareAndSet(cur, next))
+ return true;
+ }
+ }
+
+ /**
+ * Decrements the usages of the client and closes it if this is the last usage.
+ *
+ * @return {@code True} if client can be closed safely by the called.
+ */
+ public boolean release() {
+ int cnt0 = cnt.decrementAndGet();
+
+ assert cnt0 >= 0;
+
+ if (cnt0 == 0) {
+ if (cnt.compareAndSet(0, -1)) {
+ GridClient cli0 = cli;
+
+ if (cli0 != null)
+ cli0.close();
+
+ return true;
+ }
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ffe53eb5/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
index 0805be1..a4b5e6a 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolMultipleServersSelfTest.java
@@ -23,8 +23,8 @@ import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
@@ -40,13 +40,10 @@ import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteFileSystem;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider;
import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.client.GridClient;
import org.apache.ignite.internal.client.GridServerUnreachableException;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopAbstractSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
@@ -79,34 +76,12 @@ public class HadoopClientProtocolMultipleServersSelfTest extends HadoopAbstractS
}
/** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- clearClients();
- }
-
- /** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
stopAllGrids();
- clearClients();
-
super.afterTest();
}
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void clearConnectionMap() throws IgniteCheckedException {
- ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap =
- GridTestUtils.getFieldValue(IgniteHadoopClientProtocolProvider.class, "cliMap");
-
- for(IgniteInternalFuture<GridClient> fut : cliMap.values())
- fut.get().close();
-
- cliMap.clear();
- }
-
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -117,18 +92,6 @@ public class HadoopClientProtocolMultipleServersSelfTest extends HadoopAbstractS
}
/**
- *
- */
- private void clearClients() {
- ConcurrentHashMap<String, IgniteInternalFuture<GridClient>> cliMap = GridTestUtils.getFieldValue(
- IgniteHadoopClientProtocolProvider.class,
- IgniteHadoopClientProtocolProvider.class,
- "cliMap");
-
- cliMap.clear();
- }
-
- /**
* @throws Exception If failed.
*/
private void beforeJob() throws Exception {
@@ -154,26 +117,31 @@ public class HadoopClientProtocolMultipleServersSelfTest extends HadoopAbstractS
private void checkJobSubmit(Configuration conf) throws Exception {
final Job job = Job.getInstance(conf);
- job.setJobName(JOB_NAME);
+ try {
+ job.setJobName(JOB_NAME);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
- job.setInputFormatClass(TextInputFormat.class);
- job.setOutputFormatClass(OutFormat.class);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(OutFormat.class);
- job.setMapperClass(TestMapper.class);
- job.setReducerClass(TestReducer.class);
+ job.setMapperClass(TestMapper.class);
+ job.setReducerClass(TestReducer.class);
- job.setNumReduceTasks(0);
+ job.setNumReduceTasks(0);
- FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
- job.submit();
+ job.submit();
- job.waitForCompletion(false);
+ job.waitForCompletion(false);
- assert job.getStatus().getState() == JobStatus.State.SUCCEEDED : job.getStatus().getState();
+ assert job.getStatus().getState() == JobStatus.State.SUCCEEDED : job.getStatus().getState();
+ }
+ finally {
+ job.getCluster().close();
+ }
}
/**
@@ -197,18 +165,25 @@ public class HadoopClientProtocolMultipleServersSelfTest extends HadoopAbstractS
*/
@SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
public void testSingleAddress() throws Exception {
- // Don't use REST_PORT to test connection fails if the only this port is configured
- restPort = REST_PORT + 1;
+ try {
+ // Don't use REST_PORT to test connection fails if the only this port is configured
+ restPort = REST_PORT + 1;
- startGrids(gridCount());
+ startGrids(gridCount());
- GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
+ GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
@Override public Object call() throws Exception {
- checkJobSubmit(configSingleAddress());
- return null;
- }
- },
- GridServerUnreachableException.class, "Failed to connect to any of the servers in list");
+ checkJobSubmit(configSingleAddress());
+ return null;
+ }
+ },
+ GridServerUnreachableException.class, "Failed to connect to any of the servers in list");
+ }
+ finally {
+ FileSystem fs = FileSystem.get(configSingleAddress());
+
+ fs.close();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/ffe53eb5/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
index 1ef7dd0..7156a3d 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
@@ -50,7 +50,6 @@ import org.apache.ignite.IgniteFileSystem;
import org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopAbstractSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -115,7 +114,6 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
stopAllGrids();
super.afterTestsStopped();
-// IgniteHadoopClientProtocolProvider.cliMap.clear();
}
/** {@inheritDoc} */
@@ -196,43 +194,48 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
final Job job = Job.getInstance(conf);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
+ try {
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
- job.setMapperClass(TestCountingMapper.class);
- job.setReducerClass(TestCountingReducer.class);
- job.setCombinerClass(TestCountingCombiner.class);
+ job.setMapperClass(TestCountingMapper.class);
+ job.setReducerClass(TestCountingReducer.class);
+ job.setCombinerClass(TestCountingCombiner.class);
- FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
- FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
- job.submit();
+ job.submit();
- final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
+ final Counter cntr = job.getCounters().findCounter(TestCounter.COUNTER1);
- assertEquals(0, cntr.getValue());
+ assertEquals(0, cntr.getValue());
- cntr.increment(10);
+ cntr.increment(10);
- assertEquals(10, cntr.getValue());
+ assertEquals(10, cntr.getValue());
- // Transferring to map phase.
- setupLockFile.delete();
+ // Transferring to map phase.
+ setupLockFile.delete();
- // Transferring to reduce phase.
- mapLockFile.delete();
+ // Transferring to reduce phase.
+ mapLockFile.delete();
- job.waitForCompletion(false);
+ job.waitForCompletion(false);
- assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
+ assertEquals("job must end successfully", JobStatus.State.SUCCEEDED, job.getStatus().getState());
- final Counters counters = job.getCounters();
+ final Counters counters = job.getCounters();
- assertNotNull("counters cannot be null", counters);
- assertEquals("wrong counters count", 3, counters.countCounters());
- assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
- assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
- assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
+ assertNotNull("counters cannot be null", counters);
+ assertEquals("wrong counters count", 3, counters.countCounters());
+ assertEquals("wrong counter value", 15, counters.findCounter(TestCounter.COUNTER1).getValue());
+ assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER2).getValue());
+ assertEquals("wrong counter value", 3, counters.findCounter(TestCounter.COUNTER3).getValue());
+ }
+ finally {
+ job.getCluster().close();
+ }
}
/**
@@ -304,114 +307,119 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
final Job job = Job.getInstance(conf);
- job.setJobName(JOB_NAME);
+ try {
+ job.setJobName(JOB_NAME);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
- job.setMapperClass(TestMapper.class);
- job.setReducerClass(TestReducer.class);
+ job.setMapperClass(TestMapper.class);
+ job.setReducerClass(TestReducer.class);
- if (!noCombiners)
- job.setCombinerClass(TestCombiner.class);
+ if (!noCombiners)
+ job.setCombinerClass(TestCombiner.class);
- if (noReducers)
- job.setNumReduceTasks(0);
+ if (noReducers)
+ job.setNumReduceTasks(0);
- job.setInputFormatClass(TextInputFormat.class);
- job.setOutputFormatClass(TestOutputFormat.class);
+ job.setInputFormatClass(TextInputFormat.class);
+ job.setOutputFormatClass(TestOutputFormat.class);
- FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
- FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
+ FileInputFormat.setInputPaths(job, new Path(PATH_INPUT));
+ FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT));
- job.submit();
+ job.submit();
- JobID jobId = job.getJobID();
+ JobID jobId = job.getJobID();
- // Setup phase.
- JobStatus jobStatus = job.getStatus();
- checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
- assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
- assert jobStatus.getMapProgress() == 0.0f;
- assert jobStatus.getReduceProgress() == 0.0f;
+ // Setup phase.
+ JobStatus jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() >= 0.0f && jobStatus.getSetupProgress() < 1.0f;
+ assert jobStatus.getMapProgress() == 0.0f;
+ assert jobStatus.getReduceProgress() == 0.0f;
- U.sleep(2100);
+ U.sleep(2100);
- JobStatus recentJobStatus = job.getStatus();
+ JobStatus recentJobStatus = job.getStatus();
- assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() :
- "Old=" + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
+ assert recentJobStatus.getSetupProgress() > jobStatus.getSetupProgress() : "Old="
+ + jobStatus.getSetupProgress() + ", new=" + recentJobStatus.getSetupProgress();
- // Transferring to map phase.
- setupLockFile.delete();
+ // Transferring to map phase.
+ setupLockFile.delete();
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- return F.eq(1.0f, job.getStatus().getSetupProgress());
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return F.eq(1.0f, job.getStatus().getSetupProgress());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unexpected exception.", e);
+ }
}
- catch (Exception e) {
- throw new RuntimeException("Unexpected exception.", e);
- }
- }
- }, 5000L);
+ }, 5000L);
- // Map phase.
- jobStatus = job.getStatus();
- checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
- assert jobStatus.getSetupProgress() == 1.0f;
- assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
- assert jobStatus.getReduceProgress() == 0.0f;
+ // Map phase.
+ jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() >= 0.0f && jobStatus.getMapProgress() < 1.0f;
+ assert jobStatus.getReduceProgress() == 0.0f;
- U.sleep(2100);
+ U.sleep(2100);
- recentJobStatus = job.getStatus();
+ recentJobStatus = job.getStatus();
- assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() :
- "Old=" + jobStatus.getMapProgress() + ", new=" + recentJobStatus.getMapProgress();
+ assert recentJobStatus.getMapProgress() > jobStatus.getMapProgress() : "Old=" + jobStatus.getMapProgress()
+ + ", new=" + recentJobStatus.getMapProgress();
- // Transferring to reduce phase.
- mapLockFile.delete();
+ // Transferring to reduce phase.
+ mapLockFile.delete();
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- try {
- return F.eq(1.0f, job.getStatus().getMapProgress());
- }
- catch (Exception e) {
- throw new RuntimeException("Unexpected exception.", e);
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ try {
+ return F.eq(1.0f, job.getStatus().getMapProgress());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Unexpected exception.", e);
+ }
}
- }
- }, 5000L);
+ }, 5000L);
- if (!noReducers) {
- // Reduce phase.
- jobStatus = job.getStatus();
- checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
- assert jobStatus.getSetupProgress() == 1.0f;
- assert jobStatus.getMapProgress() == 1.0f;
- assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
+ if (!noReducers) {
+ // Reduce phase.
+ jobStatus = job.getStatus();
+ checkJobStatus(jobStatus, jobId, JOB_NAME, JobStatus.State.RUNNING, 0.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() == 1.0f;
+ assert jobStatus.getReduceProgress() >= 0.0f && jobStatus.getReduceProgress() < 1.0f;
- // Ensure that reduces progress increases.
- U.sleep(2100);
+ // Ensure that reduces progress increases.
+ U.sleep(2100);
- recentJobStatus = job.getStatus();
+ recentJobStatus = job.getStatus();
- assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() :
- "Old=" + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
+ assert recentJobStatus.getReduceProgress() > jobStatus.getReduceProgress() : "Old="
+ + jobStatus.getReduceProgress() + ", new=" + recentJobStatus.getReduceProgress();
- reduceLockFile.delete();
- }
+ reduceLockFile.delete();
+ }
- job.waitForCompletion(false);
+ job.waitForCompletion(false);
- jobStatus = job.getStatus();
- checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
- assert jobStatus.getSetupProgress() == 1.0f;
- assert jobStatus.getMapProgress() == 1.0f;
- assert jobStatus.getReduceProgress() == 1.0f;
+ jobStatus = job.getStatus();
+ checkJobStatus(job.getStatus(), jobId, JOB_NAME, JobStatus.State.SUCCEEDED, 1.0f);
+ assert jobStatus.getSetupProgress() == 1.0f;
+ assert jobStatus.getMapProgress() == 1.0f;
+ assert jobStatus.getReduceProgress() == 1.0f;
- dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
+ dumpIgfs(igfs, new IgfsPath(PATH_OUTPUT));
+ }
+ finally {
+ job.getCluster().close();
+ }
}
/**
@@ -517,7 +525,12 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
* Test Hadoop counters.
*/
public enum TestCounter {
- COUNTER1, COUNTER2, COUNTER3
+ /** */
+ COUNTER1,
+ /** */
+ COUNTER2,
+ /** */
+ COUNTER3
}
/**
@@ -535,6 +548,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
* Test combiner that counts invocations.
*/
public static class TestCountingCombiner extends TestReducer {
+ /** {@inheritDoc} */
@Override public void reduce(Text key, Iterable<IntWritable> values,
Context ctx) throws IOException, InterruptedException {
ctx.getCounter(TestCounter.COUNTER1).increment(1);
@@ -552,6 +566,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
* Test reducer that counts invocations.
*/
public static class TestCountingReducer extends TestReducer {
+ /** {@inheritDoc} */
@Override public void reduce(Text key, Iterable<IntWritable> values,
Context ctx) throws IOException, InterruptedException {
ctx.getCounter(TestCounter.COUNTER1).increment(1);
@@ -566,6 +581,9 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
// No-op.
}
+ /**
+ * Test output format.
+ */
public static class TestOutputFormat<K, V> extends TextOutputFormat<K, V> {
/** {@inheritDoc} */
@Override public synchronized OutputCommitter getOutputCommitter(TaskAttemptContext ctx)
[22/50] [abbrv] ignite git commit: IGNITE-4271: Hadoop: set proper
message type IDs to avoid clashing with SQL messages.
Posted by sb...@apache.org.
IGNITE-4271: Hadoop: set proper message type IDs to avoid clashing with SQL messages.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e857f297
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e857f297
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e857f297
Branch: refs/heads/master
Commit: e857f297bb22bde984b5e99c1fc62a389fa54f9b
Parents: be12a7e
Author: devozerov <vo...@gridgain.com>
Authored: Tue Dec 6 13:12:23 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:06 2016 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 16 ++++++++--------
.../internal/processors/hadoop/HadoopJobId.java | 2 +-
.../processors/hadoop/shuffle/HadoopShuffleAck.java | 2 +-
.../hadoop/shuffle/HadoopShuffleMessage.java | 2 +-
4 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e857f297/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index dd68984..86742e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -168,21 +168,26 @@ public class GridIoMessageFactory implements MessageFactory {
Message msg = null;
switch (type) {
- case -30:
+ case -39:
msg = new HadoopJobId();
break;
- case -29:
+ case -38:
msg = new HadoopShuffleAck();
break;
- case -28:
+ case -37:
msg = new HadoopShuffleMessage();
break;
+ case -36:
+ msg = new GridDhtAtomicSingleUpdateRequest();
+
+ break;
+
case -27:
msg = new GridDhtTxOnePhaseCommitAckRequest();
@@ -793,11 +798,6 @@ public class GridIoMessageFactory implements MessageFactory {
break;
- case -36:
- msg = new GridDhtAtomicSingleUpdateRequest();
-
- break;
-
// [-3..119] [124..127] [-36]- this
// [120..123] - DR
// [-4..-22, -30..-35] - SQL
http://git-wip-us.apache.org/repos/asf/ignite/blob/e857f297/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
index 740ab89..761b904 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
@@ -126,7 +126,7 @@ public class HadoopJobId implements Message, GridCacheInternal, Externalizable {
/** {@inheritDoc} */
@Override public byte directType() {
- return -30;
+ return -39;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/e857f297/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
index 6dd2c2d..9a03f99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
@@ -136,7 +136,7 @@ public class HadoopShuffleAck implements HadoopMessage, Message {
/** {@inheritDoc} */
@Override public byte directType() {
- return -29;
+ return -38;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/e857f297/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
index 3732bc2..af54dcc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -302,7 +302,7 @@ public class HadoopShuffleMessage implements Message, HadoopMessage {
/** {@inheritDoc} */
@Override public byte directType() {
- return -28;
+ return -37;
}
/** {@inheritDoc} */
[12/50] [abbrv] ignite git commit: IGNITE-3862 - GridServiceProxy
invocation never times out. Fix
Posted by sb...@apache.org.
IGNITE-3862 - GridServiceProxy invocation never times out. Fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/eeb2f2a3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/eeb2f2a3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/eeb2f2a3
Branch: refs/heads/master
Commit: eeb2f2a38fb792b711eb665e380d14bc00f6e078
Parents: 6d744db
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Dec 12 12:14:01 2016 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Dec 12 12:14:01 2016 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/IgniteServices.java | 18 +-
.../ignite/internal/IgniteServicesImpl.java | 11 +-
.../affinity/GridAffinityProcessor.java | 2 +-
.../processors/cache/GridCacheAdapter.java | 6 +-
.../CacheDataStructuresManager.java | 6 +-
.../cache/query/GridCacheQueryManager.java | 4 +-
.../closure/GridClosureProcessor.java | 32 ++-
.../internal/processors/job/GridJobWorker.java | 7 +
.../platform/services/PlatformServices.java | 2 +-
.../service/GridServiceProcessor.java | 11 +-
.../processors/service/GridServiceProxy.java | 18 +-
.../processors/task/GridTaskWorker.java | 7 +
.../IgniteComputeTopologyExceptionTest.java | 5 +-
...gniteServiceProxyTimeoutInitializedTest.java | 284 +++++++++++++++++++
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
15 files changed, 390 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
index 08577c5..83fd487 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java
@@ -350,6 +350,22 @@ public interface IgniteServices extends IgniteAsyncSupport {
*/
public <T> T serviceProxy(String name, Class<? super T> svcItf, boolean sticky) throws IgniteException;
+ /**
+ * Gets a remote handle on the service with timeout. If service is available locally,
+ * then local instance is returned and timeout ignored, otherwise, a remote proxy is dynamically
+ * created and provided for the specified service.
+ *
+ * @param name Service name.
+ * @param svcItf Interface for the service.
+ * @param sticky Whether or not Ignite should always contact the same remote
+ * service or try to load-balance between services.
+ * @param timeout If greater than 0 created proxy will wait for service availability only specified time,
+ * and will limit remote service invocation time.
+ * @return Either proxy over remote service or local service if it is deployed locally.
+ * @throws IgniteException If failed to create service proxy.
+ */
+ public <T> T serviceProxy(String name, Class<? super T> svcItf, boolean sticky, long timeout) throws IgniteException;
+
/** {@inheritDoc} */
@Override public IgniteServices withAsync();
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
index b8042c3..400f28d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteServicesImpl.java
@@ -222,14 +222,21 @@ public class IgniteServicesImpl extends AsyncSupportAdapter implements IgniteSer
/** {@inheritDoc} */
@Override public <T> T serviceProxy(String name, Class<? super T> svcItf, boolean sticky)
throws IgniteException {
+ return (T) serviceProxy(name, svcItf, sticky, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> T serviceProxy(final String name, final Class<? super T> svcItf, final boolean sticky,
+ final long timeout) throws IgniteException {
A.notNull(name, "name");
A.notNull(svcItf, "svcItf");
A.ensure(svcItf.isInterface(), "Service class must be an interface: " + svcItf);
+ A.ensure(timeout >= 0, "Timeout cannot be negative: " + timeout);
guard();
try {
- return (T)ctx.service().serviceProxy(prj, name, svcItf, sticky);
+ return (T)ctx.service().serviceProxy(prj, name, svcItf, sticky, timeout);
}
finally {
unguard();
@@ -289,4 +296,4 @@ public class IgniteServicesImpl extends AsyncSupportAdapter implements IgniteSer
protected Object readResolve() throws ObjectStreamException {
return prj.services();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
index b9182ae..b6efafb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessor.java
@@ -497,7 +497,7 @@ public class GridAffinityProcessor extends GridProcessorAdapter {
private AffinityInfo affinityInfoFromNode(@Nullable String cacheName, AffinityTopologyVersion topVer, ClusterNode n)
throws IgniteCheckedException {
GridTuple3<GridAffinityMessage, GridAffinityMessage, GridAffinityAssignment> t = ctx.closure()
- .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/).get();
+ .callAsyncNoFailover(BROADCAST, affinityJob(cacheName, topVer), F.asList(n), true/*system pool*/, 0).get();
AffinityFunction f = (AffinityFunction)unmarshall(ctx, n.id(), t.get1());
AffinityKeyMapper m = (AffinityKeyMapper)unmarshall(ctx, n.id(), t.get2());
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index b30ec70..4d59d50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3628,13 +3628,15 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return ctx.closures().callAsyncNoFailover(BROADCAST,
new LoadKeysCallableV2<>(ctx.name(), keys, update, plc, keepBinary),
nodes,
- true);
+ true,
+ 0);
}
else {
return ctx.closures().callAsyncNoFailover(BROADCAST,
new LoadKeysCallable<>(ctx.name(), keys, update, plc),
nodes,
- true);
+ true,
+ 0);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
index c018f71..c1983df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/datastructures/CacheDataStructuresManager.java
@@ -491,7 +491,8 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
cctx.closures().callAsyncNoFailover(BROADCAST,
new BlockSetCallable(cctx.name(), id),
nodes,
- true).get();
+ true,
+ 0).get();
}
catch (IgniteCheckedException e) {
if (e.hasCause(ClusterTopologyCheckedException.class)) {
@@ -514,7 +515,8 @@ public class CacheDataStructuresManager extends GridCacheManagerAdapter {
cctx.closures().callAsyncNoFailover(BROADCAST,
new RemoveSetDataCallable(cctx.name(), id, topVer),
nodes,
- true).get();
+ true,
+ 0).get();
}
catch (IgniteCheckedException e) {
if (e.hasCause(ClusterTopologyCheckedException.class)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index d4decb4..1165157 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -220,7 +220,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
if (detailMetricsSz > 0)
detailMetrics = new ConcurrentHashMap8<>(detailMetricsSz);
-
+
lsnr = new GridLocalEventListener() {
@Override public void onEvent(Event evt) {
UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
@@ -2237,7 +2237,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
// Get metadata from remote nodes.
if (!nodes.isEmpty())
- rmtFut = cctx.closures().callAsyncNoFailover(BROADCAST, Collections.singleton(job), nodes, true);
+ rmtFut = cctx.closures().callAsyncNoFailover(BROADCAST, Collections.singleton(job), nodes, true, 0);
// Get local metadata.
IgniteInternalFuture<Collection<CacheSqlMetadata>> locFut = cctx.closures().callLocalSafe(job, true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 9d295d3..3ed985e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -81,6 +81,7 @@ import static org.apache.ignite.compute.ComputeJobResultPolicy.FAILOVER;
import static org.apache.ignite.compute.ComputeJobResultPolicy.REDUCE;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
+import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_TIMEOUT;
/**
*
@@ -520,9 +521,15 @@ public class GridClosureProcessor extends GridProcessorAdapter {
* @param <R> Type.
* @return Grid future for collection of closure results.
*/
- public <R> IgniteInternalFuture<R> callAsyncNoFailover(GridClosureCallMode mode, @Nullable Callable<R> job,
- @Nullable Collection<ClusterNode> nodes, boolean sys) {
+ public <R> IgniteInternalFuture<R> callAsyncNoFailover(
+ GridClosureCallMode mode,
+ @Nullable Callable<R> job,
+ @Nullable Collection<ClusterNode> nodes,
+ boolean sys,
+ long timeout
+ ) {
assert mode != null;
+ assert timeout >= 0 : timeout;
busyLock.readLock();
@@ -536,6 +543,9 @@ public class GridClosureProcessor extends GridProcessorAdapter {
ctx.task().setThreadContext(TC_NO_FAILOVER, true);
ctx.task().setThreadContext(TC_SUBGRID, nodes);
+ if (timeout > 0)
+ ctx.task().setThreadContext(TC_TIMEOUT, timeout);
+
return ctx.task().execute(new T7<>(mode, job), null, sys);
}
finally {
@@ -548,13 +558,19 @@ public class GridClosureProcessor extends GridProcessorAdapter {
* @param jobs Closures to execute.
* @param nodes Grid nodes.
* @param sys If {@code true}, then system pool will be used.
+ * @param timeout If greater than 0 limits task execution. Cannot be negative.
* @param <R> Type.
* @return Grid future for collection of closure results.
*/
- public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(GridClosureCallMode mode,
- @Nullable Collection<? extends Callable<R>> jobs, @Nullable Collection<ClusterNode> nodes,
- boolean sys) {
+ public <R> IgniteInternalFuture<Collection<R>> callAsyncNoFailover(
+ GridClosureCallMode mode,
+ @Nullable Collection<? extends Callable<R>> jobs,
+ @Nullable Collection<ClusterNode> nodes,
+ boolean sys,
+ long timeout
+ ) {
assert mode != null;
+ assert timeout >= 0 : timeout;
busyLock.readLock();
@@ -568,6 +584,9 @@ public class GridClosureProcessor extends GridProcessorAdapter {
ctx.task().setThreadContext(TC_NO_FAILOVER, true);
ctx.task().setThreadContext(TC_SUBGRID, nodes);
+ if (timeout > 0)
+ ctx.task().setThreadContext(TC_TIMEOUT, timeout);
+
return ctx.task().execute(new T6<>(mode, jobs), null, sys);
}
finally {
@@ -580,6 +599,7 @@ public class GridClosureProcessor extends GridProcessorAdapter {
* @param job Closure to execute.
* @param nodes Grid nodes.
* @param sys If {@code true}, then system pool will be used.
+ * @param timeout If greater than 0 limits task execution. Cannot be negative.
* @param <R> Type.
* @return Grid future for collection of closure results.
*/
@@ -2304,4 +2324,4 @@ public class GridClosureProcessor extends GridProcessorAdapter {
return S.toString(C4MLAV2.class, this, super.toString());
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index f5c6a27..9bee849 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -43,14 +43,17 @@ import org.apache.ignite.internal.GridJobExecuteResponse;
import org.apache.ignite.internal.GridJobSessionImpl;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.service.GridServiceNotFoundException;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
@@ -587,6 +590,10 @@ public class GridJobWorker extends GridWorker implements GridTimeoutObject {
else
U.warn(log, msg);
}
+ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
+ X.hasCause(e, ClusterTopologyCheckedException.class))
+ // Should be throttled, because GridServiceProxy continuously retry getting service.
+ LT.error(log, e, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']');
else
U.error(log, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']', e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index 962a4c0..c266986 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@ -340,7 +340,7 @@ public class PlatformServices extends PlatformAbstractTarget {
Object proxy = PlatformService.class.isAssignableFrom(d.serviceClass())
? services.serviceProxy(name, PlatformService.class, sticky)
- : new GridServiceProxy<>(services.clusterGroup(), name, Service.class, sticky,
+ : new GridServiceProxy<>(services.clusterGroup(), name, Service.class, sticky, 0,
platformCtx.kernalContext());
return new ServiceProxyHolder(proxy, d.serviceClass());
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index b9b92b8..3690f35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -694,9 +694,10 @@ public class GridServiceProcessor extends GridProcessorAdapter {
/**
* @param name Service name.
+ * @param timeout If greater than 0 limits task execution time. Cannot be negative.
* @return Service topology.
*/
- public Map<UUID, Integer> serviceTopology(String name) throws IgniteCheckedException {
+ public Map<UUID, Integer> serviceTopology(String name, long timeout) throws IgniteCheckedException {
ClusterNode node = cache.affinity().mapKeyToNode(name);
if (node.version().compareTo(ServiceTopologyCallable.SINCE_VER) >= 0) {
@@ -708,7 +709,8 @@ public class GridServiceProcessor extends GridProcessorAdapter {
GridClosureCallMode.BROADCAST,
call,
Collections.singletonList(node),
- false
+ false,
+ timeout
).get();
}
else
@@ -828,12 +830,13 @@ public class GridServiceProcessor extends GridProcessorAdapter {
* @param name Service name.
* @param svcItf Service class.
* @param sticky Whether multi-node request should be done.
+ * @param timeout If greater than 0 limits service acquire time. Cannot be negative.
* @param <T> Service interface type.
* @return The proxy of a service by its name and class.
* @throws IgniteException If failed to create proxy.
*/
@SuppressWarnings("unchecked")
- public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf, boolean sticky)
+ public <T> T serviceProxy(ClusterGroup prj, String name, Class<? super T> svcItf, boolean sticky, long timeout)
throws IgniteException {
if (hasLocalNode(prj)) {
ServiceContextImpl ctx = serviceContext(name);
@@ -851,7 +854,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
}
}
- return new GridServiceProxy<T>(prj, name, svcItf, sticky, ctx).proxy();
+ return new GridServiceProxy<T>(prj, name, svcItf, sticky, timeout, ctx).proxy();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index 564a13a..aa60934 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -84,11 +84,15 @@ public class GridServiceProxy<T> implements Serializable {
/** Whether multi-node request should be done. */
private final boolean sticky;
+ /** Service availability wait timeout. */
+ private final long waitTimeout;
+
/**
* @param prj Grid projection.
* @param name Service name.
* @param svc Service type class.
* @param sticky Whether multi-node request should be done.
+ * @param timeout Service availability wait timeout. Cannot be negative.
* @param ctx Context.
*/
@SuppressWarnings("unchecked")
@@ -96,12 +100,16 @@ public class GridServiceProxy<T> implements Serializable {
String name,
Class<? super T> svc,
boolean sticky,
+ long timeout,
GridKernalContext ctx)
{
+ assert timeout >= 0 : timeout;
+
this.prj = prj;
this.ctx = ctx;
this.name = name;
this.sticky = sticky;
+ this.waitTimeout = timeout;
hasLocNode = hasLocalNode(prj);
log = ctx.log(getClass());
@@ -145,6 +153,8 @@ public class GridServiceProxy<T> implements Serializable {
ctx.gateway().readLock();
try {
+ final long startTime = U.currentTimeMillis();
+
while (true) {
ClusterNode node = null;
@@ -171,7 +181,8 @@ public class GridServiceProxy<T> implements Serializable {
GridClosureCallMode.BROADCAST,
new ServiceProxyCallable(mtd.getName(), name, mtd.getParameterTypes(), args),
Collections.singleton(node),
- false
+ false,
+ waitTimeout
).get();
}
}
@@ -203,6 +214,9 @@ public class GridServiceProxy<T> implements Serializable {
throw new IgniteException(e);
}
+
+ if (waitTimeout > 0 && U.currentTimeMillis() - startTime >= waitTimeout)
+ throw new IgniteException("Service acquire timeout was reached, stopping. [timeout=" + waitTimeout + "]");
}
}
finally {
@@ -246,7 +260,7 @@ public class GridServiceProxy<T> implements Serializable {
if (hasLocNode && ctx.service().service(name) != null)
return ctx.discovery().localNode();
- Map<UUID, Integer> snapshot = ctx.service().serviceTopology(name);
+ Map<UUID, Integer> snapshot = ctx.service().serviceTopology(name, waitTimeout);
if (snapshot == null || snapshot.isEmpty())
return null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
index 3478c70..d89e80b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskWorker.java
@@ -68,6 +68,7 @@ import org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException;
import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.closure.AffinityTask;
+import org.apache.ignite.internal.processors.service.GridServiceNotFoundException;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.typedef.CO;
import org.apache.ignite.internal.util.typedef.F;
@@ -1065,6 +1066,12 @@ class GridTaskWorker<T, R> extends GridWorker implements GridTimeoutObject {
return null;
}
+ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
+ X.hasCause(e, ClusterTopologyCheckedException.class)) {
+ // Should be throttled, because GridServiceProxy continuously retry getting service.
+ LT.error(log, e, "Failed to obtain remote job result policy for result from " +
+ "ComputeTask.result(..) method (will fail the whole task): " + jobRes);
+ }
else
U.error(log, "Failed to obtain remote job result policy for result from " +
"ComputeTask.result(..) method (will fail the whole task): " + jobRes, e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
index c74daca..3ed91e8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeTopologyExceptionTest.java
@@ -88,7 +88,8 @@ public class IgniteComputeTopologyExceptionTest extends GridCommonAbstractTest {
}
},
nodes,
- false);
+ false,
+ 0);
try {
fut.get();
@@ -99,4 +100,4 @@ public class IgniteComputeTopologyExceptionTest extends GridCommonAbstractTest {
log.info("Expected exception: " + e);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java
new file mode 100644
index 0000000..41eef31
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceProxyTimeoutInitializedTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.binary.BinaryBasicIdMapper;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryReader;
+import org.apache.ignite.binary.BinaryWriter;
+import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeTaskTimeoutException;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests service proxy timeouts.
+ */
+public class IgniteServiceProxyTimeoutInitializedTest extends GridCommonAbstractTest {
+ /** */
+ private static Service srvc;
+
+ /** */
+ private static CountDownLatch latch1;
+
+ /** */
+ private static CountDownLatch latch2;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ final ServiceConfiguration scfg = new ServiceConfiguration();
+
+ if (gridName.endsWith("0")) {
+ scfg.setName("testService");
+ scfg.setService(srvc);
+ scfg.setMaxPerNodeCount(1);
+ scfg.setTotalCount(1);
+ scfg.setNodeFilter(new NodeFilter());
+
+ final Map<String, String> attrs = new HashMap<>();
+
+ attrs.put("clusterGroup", "0");
+
+ cfg.setUserAttributes(attrs);
+
+ cfg.setServiceConfiguration(scfg);
+ }
+
+ cfg.setMarshaller(null);
+
+ final BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ // Despite defaults explicitly set to lower case.
+ binCfg.setIdMapper(new BinaryBasicIdMapper(true));
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+
+ /**
+ * Checks that we limit retries to get not available service by timeout.
+ *
+ * @throws Exception If fail.
+ */
+ @SuppressWarnings({"Convert2Lambda", "ThrowableResultOfMethodCallIgnored"})
+ public void testUnavailableService() throws Exception {
+ srvc = new TestWaitServiceImpl();
+
+ latch1 = new CountDownLatch(1);
+ latch2 = new CountDownLatch(1);
+
+ try {
+ GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ startGrid(0);
+
+ return null;
+ }
+ });
+
+ assert latch1.await(1, TimeUnit.MINUTES);
+
+ final IgniteEx ignite1 = startGrid(1);
+
+ final TestService testSrvc = ignite1.services().serviceProxy("testService", TestService.class, false, 500);
+
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ testSrvc.test();
+
+ return null;
+ }
+ }, IgniteException.class, null);
+ }
+ finally {
+ latch2.countDown();
+ }
+ }
+
+ /**
+ * Checks that service not hangs if timeout set. Here we get hang with marshalling exception.
+ *
+ * @throws Exception If fail.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+ public void testServiceException() throws Exception {
+ srvc = new HangServiceImpl();
+
+ // Start service grid.
+ startGrid(0);
+ final IgniteEx ignite1 = startGrid(1);
+
+ final HangService testSrvc = ignite1.services().serviceProxy("testService", HangService.class, false, 1_000);
+
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ testSrvc.hang();
+
+ return null;
+ }
+ }, ComputeTaskTimeoutException.class, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 60_000;
+ }
+
+ /**
+ *
+ */
+ private static class NodeFilter implements IgnitePredicate<ClusterNode> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public boolean apply(final ClusterNode clusterNode) {
+ return "0".equals(clusterNode.attribute("clusterGroup"));
+ }
+ }
+
+ /**
+ *
+ */
+ private interface TestService {
+ /** */
+ void test();
+ }
+
+ /**
+ *
+ */
+ private static class TestWaitServiceImpl implements Service, TestService {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public void test() {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel(final ServiceContext ctx) {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(final ServiceContext ctx) throws Exception {
+ latch1.countDown();
+
+ // Simulate long initialization.
+ latch2.await(1, TimeUnit.MINUTES);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute(final ServiceContext ctx) throws Exception {
+ // No-op
+ }
+ }
+
+ /**
+ *
+ */
+ private static class HangClass implements Binarylizable {
+
+ /** {@inheritDoc} */
+ @Override public void writeBinary(final BinaryWriter writer) throws BinaryObjectException {
+ try {
+ U.sleep(10_000);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new BinaryObjectException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readBinary(final BinaryReader reader) throws BinaryObjectException {
+ try {
+ U.sleep(10_000);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new BinaryObjectException(e);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private interface HangService {
+ /**
+ * @return Hangs deserialization.
+ */
+ HangClass hang();
+ }
+
+ /**
+ *
+ */
+ private static class HangServiceImpl implements HangService, Service {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public HangClass hang() {
+ return new HangClass();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel(final ServiceContext ctx) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(final ServiceContext ctx) throws Exception {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute(final ServiceContext ctx) throws Exception {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/eeb2f2a3/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index d9cc8c0..350b715 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -66,6 +66,7 @@ import org.apache.ignite.internal.processors.service.IgniteServiceDeployment2Cla
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingDefaultMarshallerTest;
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingJdkMarshallerTest;
import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest;
+import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
@@ -141,6 +142,7 @@ public class IgniteKernalSelfTestSuite extends TestSuite {
suite.addTestSuite(GridServiceProxyNodeStopSelfTest.class);
suite.addTestSuite(GridServiceProxyClientReconnectSelfTest.class);
suite.addTestSuite(IgniteServiceReassignmentTest.class);
+ suite.addTestSuite(IgniteServiceProxyTimeoutInitializedTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class);
[32/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
index 482e2ef..c7a1a53 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiMBean.java
@@ -17,6 +17,7 @@
package org.apache.ignite.spi.communication.tcp;
+import org.apache.ignite.internal.util.nio.GridNioServer;
import org.apache.ignite.mxbean.MXBeanDescription;
import org.apache.ignite.spi.IgniteSpiManagementMBean;
@@ -44,6 +45,35 @@ public interface TcpCommunicationSpiMBean extends IgniteSpiManagementMBean {
public int getLocalPort();
/**
+ * Returns {@code true} if {@code TcpCommunicationSpi} should
+ * maintain connection for outgoing and incoming messages separately.
+ * In this case total number of connections between local and some remote node
+ * is {@link #getConnectionsPerNode()} * 2.
+ * <p>
+ * Returns {@code false} if each connection of {@link #getConnectionsPerNode()}
+ * should be used for outgoing and incoming messages. In this case load NIO selectors load
+ * balancing of {@link GridNioServer} will be disabled.
+ * <p>
+ * Default is {@code true}.
+ *
+ * @return {@code true} to use paired connections and {@code false} otherwise.
+ * @see #getConnectionsPerNode()
+ */
+ @MXBeanDescription("Paired connections used.")
+ public boolean isUsePairedConnections();
+
+ /**
+ * Gets number of connections to each remote node. if {@link #isUsePairedConnections()}
+ * is {@code true} then number of connections is doubled and half is used for incoming and
+ * half for outgoing messages.
+ *
+ * @return Number of connections per node.
+ * @see #isUsePairedConnections()
+ */
+ @MXBeanDescription("Connections per node.")
+ public int getConnectionsPerNode();
+
+ /**
* Gets local port for shared memory communication.
*
* @return Port number.
@@ -153,6 +183,16 @@ public interface TcpCommunicationSpiMBean extends IgniteSpiManagementMBean {
public int getReconnectCount();
/**
+ * Defines how many non-blocking {@code selector.selectNow()} should be made before
+ * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+ * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+ *
+ * @return Selector thread busy-loop iterations.
+ */
+ @MXBeanDescription("Selector thread busy-loop iterations.")
+ public long getSelectorSpins();
+
+ /**
* Gets value for {@code TCP_NODELAY} socket option.
*
* @return {@code True} if TCP delay is disabled.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 9a36f1a..8a9f1c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3408,7 +3408,7 @@ class ServerImpl extends TcpDiscoveryImpl {
if (log.isDebugEnabled())
log.debug("Node validation failed [res=" + err + ", node=" + node + ']');
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
boolean ping = node.id().equals(err.nodeId()) ? pingNode(node) : pingNode(err.nodeId());
@@ -3453,7 +3453,7 @@ class ServerImpl extends TcpDiscoveryImpl {
final String rmtMarsh = node.attribute(ATTR_MARSHALLER);
if (!F.eq(locMarsh, rmtMarsh)) {
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
String errMsg = "Local node's marshaller differs from remote node's marshaller " +
@@ -3510,7 +3510,7 @@ class ServerImpl extends TcpDiscoveryImpl {
boolean locLateAssignBool = locLateAssign != null ? locLateAssign : false;
if (locMarshUseDfltSuidBool != rmtMarshUseDfltSuidBool) {
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
String errMsg = "Local node's " + IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID +
@@ -3552,7 +3552,7 @@ class ServerImpl extends TcpDiscoveryImpl {
final boolean rmtMarshCompactFooterBool = rmtMarshCompactFooter != null ? rmtMarshCompactFooter : false;
if (locMarshCompactFooterBool != rmtMarshCompactFooterBool) {
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
String errMsg = "Local node's binary marshaller \"compactFooter\" property differs from " +
@@ -3590,7 +3590,7 @@ class ServerImpl extends TcpDiscoveryImpl {
final boolean rmtMarshStrSerialVer2Bool = rmtMarshStrSerialVer2 != null ? rmtMarshStrSerialVer2 : false;
if (locMarshStrSerialVer2Bool != rmtMarshStrSerialVer2Bool) {
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
String errMsg = "Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 +
@@ -3663,7 +3663,7 @@ class ServerImpl extends TcpDiscoveryImpl {
final Boolean rmtSrvcCompatibilityEnabled = node.attribute(ATTR_SERVICES_COMPATIBILITY_MODE);
if (!F.eq(locSrvcCompatibilityEnabled, rmtSrvcCompatibilityEnabled)) {
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
String errMsg = "Local node's " + IGNITE_SERVICES_COMPATIBILITY_MODE +
@@ -3698,7 +3698,7 @@ class ServerImpl extends TcpDiscoveryImpl {
}
}
else if (Boolean.FALSE.equals(locSrvcCompatibilityEnabled)) {
- utilityPool.submit(
+ utilityPool.execute(
new Runnable() {
@Override public void run() {
String errMsg = "Remote node doesn't support lazy services configuration and " +
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java b/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
index d1c8d19..127778b 100644
--- a/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
+++ b/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
@@ -184,6 +184,7 @@ public class SocketStreamer<T, K, V> extends StreamAdapter<T, K, V> {
try {
srv = new GridNioServer.Builder<byte[]>()
.address(addr == null ? InetAddress.getLocalHost() : addr)
+ .serverName("sock-streamer")
.port(port)
.listener(lsnr)
.logger(log)
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
index 55557dd..d173594 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
@@ -20,6 +20,7 @@ package org.apache.ignite.thread;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.NotNull;
/**
@@ -62,4 +63,9 @@ public class IgniteThreadFactory implements ThreadFactory {
@Override public Thread newThread(@NotNull Runnable r) {
return new IgniteThread(gridName, threadName, r, idxGen.incrementAndGet());
}
-}
\ No newline at end of file
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgniteThreadFactory.class, this, super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
index 760313b..5721887 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteSlowClientDetectionSelfTest.java
@@ -75,6 +75,7 @@ public class IgniteSlowClientDetectionSelfTest extends GridCommonAbstractTest {
commSpi.setSlowClientQueueLimit(50);
commSpi.setSharedMemoryPort(-1);
commSpi.setIdleConnectionTimeout(300_000);
+ commSpi.setConnectionsPerNode(1);
cfg.setCommunicationSpi(commSpi);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java
new file mode 100644
index 0000000..e95b1ec
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+/**
+ *
+ */
+public class IgniteCommunicationBalanceMultipleConnectionsTest extends IgniteCommunicationBalanceTest {
+ /** {@inheritDoc} */
+ @Override protected int connectionsPerNode() {
+ return 5;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
new file mode 100644
index 0000000..e142aef
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceTest.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
+import org.apache.ignite.internal.util.nio.GridNioServer;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteCommunicationBalanceTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private boolean client;
+
+ /** */
+ private int selectors;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpCommunicationSpi commSpi = ((TcpCommunicationSpi)cfg.getCommunicationSpi());
+
+ commSpi.setSharedMemoryPort(-1);
+ commSpi.setConnectionsPerNode(connectionsPerNode());
+
+ if (selectors > 0)
+ commSpi.setSelectorsCount(selectors);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ cfg.setClientMode(client);
+
+ return cfg;
+ }
+
+ /**
+ * @return Connections per node.
+ */
+ protected int connectionsPerNode() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBalance1() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "5000");
+
+ try {
+ selectors = 4;
+
+ final int SRVS = 4;
+
+ startGridsMultiThreaded(SRVS);
+
+ client = true;
+
+ final Ignite client = startGrid(SRVS);
+
+ for (int i = 0; i < 4; i++) {
+ ClusterNode node = client.cluster().node(ignite(i).cluster().localNode().id());
+
+ client.compute(client.cluster().forNode(node)).call(new DummyCallable(null));
+ }
+
+ waitNioBalanceStop(Collections.singletonList(client), 10_000);
+
+ final GridNioServer srv = GridTestUtils.getFieldValue(client.configuration().getCommunicationSpi(), "nioSrvr");
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ long readMoveCnt1 = srv.readerMoveCount();
+ long writeMoveCnt1 = srv.writerMoveCount();
+
+ int prevNodeIdx = -1;
+
+ for (int iter = 0; iter < 10; iter++) {
+ int nodeIdx = rnd.nextInt(SRVS);
+
+ while (prevNodeIdx == nodeIdx)
+ nodeIdx = rnd.nextInt(SRVS);
+
+ prevNodeIdx = nodeIdx;
+
+ log.info("Iteration [iter=" + iter + ", node=" + nodeIdx + ']');
+
+ final long readMoveCnt = readMoveCnt1;
+ final long writeMoveCnt = writeMoveCnt1;
+
+ final int nodeIdx0 = nodeIdx;
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ byte[] data = new byte[100_000];
+
+ for (int j = 0; j < 10; j++) {
+ for (int i = 0; i < SRVS; i++) {
+ ClusterNode node = client.cluster().node(ignite(i).cluster().localNode().id());
+
+ IgniteCompute compute = client.compute(client.cluster().forNode(node));
+
+ compute.call(new DummyCallable(i == nodeIdx0 ? data : null));
+ }
+ }
+
+ return srv.readerMoveCount() > readMoveCnt && srv.writerMoveCount() > writeMoveCnt;
+ }
+ }, 30_000);
+
+ waitNioBalanceStop(Collections.singletonList(client), 30_000);
+
+ long readMoveCnt2 = srv.readerMoveCount();
+ long writeMoveCnt2 = srv.writerMoveCount();
+
+ log.info("Move counts [rc1=" + readMoveCnt1 +
+ ", wc1=" + writeMoveCnt1 +
+ ", rc2=" + readMoveCnt2 +
+ ", wc2=" + writeMoveCnt2 + ']');
+
+ assertTrue(readMoveCnt2 > readMoveCnt1);
+ assertTrue(writeMoveCnt2 > writeMoveCnt1);
+
+ readMoveCnt1 = readMoveCnt2;
+ writeMoveCnt1 = writeMoveCnt2;
+ }
+
+ waitNioBalanceStop(G.allGrids(), 10_000);
+ }
+ finally {
+ System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "");
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBalance2() throws Exception {
+ System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "1000");
+
+ try {
+ startGridsMultiThreaded(5);
+
+ client = true;
+
+ startGridsMultiThreaded(5, 5);
+
+ for (int i = 0; i < 5; i++) {
+ log.info("Iteration: " + i);
+
+ final AtomicInteger idx = new AtomicInteger();
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Ignite node = ignite(idx.incrementAndGet() % 10);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int msgs = rnd.nextInt(500, 600);
+
+ for (int i = 0; i < msgs; i++) {
+ int sndTo = rnd.nextInt(10);
+
+ ClusterNode sntToNode = node.cluster().node(ignite(sndTo).cluster().localNode().id());
+
+ IgniteCompute compute = node.compute(node.cluster().forNode(sntToNode));
+
+ compute.call(new DummyCallable(new byte[rnd.nextInt(rnd.nextInt(256, 1024))]));
+ }
+
+ return null;
+ }
+ }, 30, "test-thread");
+
+ waitNioBalanceStop(G.allGrids(), 10_000);
+ }
+ }
+ finally {
+ System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "");
+ }
+ }
+
+ /**
+ * @param nodes Node.
+ * @param timeout Timeout.
+ * @throws Exception If failed.
+ */
+ private void waitNioBalanceStop(List<Ignite> nodes, long timeout) throws Exception {
+ final List<GridNioServer> srvs = new ArrayList<>();
+
+ for (Ignite node : nodes) {
+ TcpCommunicationSpi spi = (TcpCommunicationSpi) node.configuration().getCommunicationSpi();
+
+ GridNioServer srv = GridTestUtils.getFieldValue(spi, "nioSrvr");
+
+ srvs.add(srv);
+ }
+
+ assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicateX() {
+ @Override public boolean applyx() throws IgniteCheckedException {
+ List<Long> rCnts = new ArrayList<>();
+ List<Long> wCnts = new ArrayList<>();
+
+ for (GridNioServer srv : srvs) {
+ long readerMovCnt1 = srv.readerMoveCount();
+ long writerMovCnt1 = srv.writerMoveCount();
+
+ rCnts.add(readerMovCnt1);
+ wCnts.add(writerMovCnt1);
+ }
+
+ U.sleep(2000);
+
+ for (int i = 0; i < srvs.size(); i++) {
+ GridNioServer srv = srvs.get(i);
+
+ long readerMovCnt1 = rCnts.get(i);
+ long writerMovCnt1 = wCnts.get(i);
+
+ long readerMovCnt2 = srv.readerMoveCount();
+ long writerMovCnt2 = srv.writerMoveCount();
+
+ if (readerMovCnt1 != readerMovCnt2) {
+ log.info("Readers balance is in progress [node=" + i + ", cnt1=" + readerMovCnt1 +
+ ", cnt2=" + readerMovCnt2 + ']');
+
+ return false;
+ }
+ if (writerMovCnt1 != writerMovCnt2) {
+ log.info("Writers balance is in progress [node=" + i + ", cnt1=" + writerMovCnt1 +
+ ", cnt2=" + writerMovCnt2 + ']');
+
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }, timeout));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRandomBalance() throws Exception {
+ System.setProperty(GridNioServer.IGNITE_IO_BALANCE_RANDOM_BALANCE, "true");
+ System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "500");
+
+ try {
+ final int NODES = 10;
+
+ startGridsMultiThreaded(NODES);
+
+ final long stopTime = System.currentTimeMillis() + 60_000;
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ while (System.currentTimeMillis() < stopTime)
+ ignite(rnd.nextInt(NODES)).compute().broadcast(new DummyCallable(null));
+
+ return null;
+ }
+ }, 20, "test-thread");
+ }
+ finally {
+ System.setProperty(GridNioServer.IGNITE_IO_BALANCE_RANDOM_BALANCE, "");
+ System.setProperty(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, "");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class DummyCallable implements IgniteCallable<Object> {
+ /** */
+ private byte[] data;
+
+ /**
+ * @param data Data.
+ */
+ DummyCallable(byte[] data) {
+ this.data = data;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object call() throws Exception {
+ return data;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java
new file mode 100644
index 0000000..b644878
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessagesTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteIoTestMessagesTest extends GridCommonAbstractTest {
+ /** */
+ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private boolean client;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ cfg.setClientMode(client);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(3);
+
+ client = true;
+
+ startGrid(3);
+
+ startGrid(4);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIoTestMessages() throws Exception {
+ for (Ignite node : G.allGrids()) {
+ IgniteKernal ignite = (IgniteKernal)node;
+
+ List<ClusterNode> rmts = new ArrayList<>(ignite.cluster().forRemotes().nodes());
+
+ assertEquals(4, rmts.size());
+
+ for (ClusterNode rmt : rmts) {
+ ignite.sendIoTest(rmt, new byte[1024], false);
+
+ ignite.sendIoTest(rmt, new byte[1024], true);
+
+ ignite.sendIoTest(rmts, new byte[1024], false);
+
+ ignite.sendIoTest(rmts, new byte[1024], true);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java
new file mode 100644
index 0000000..510751e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteVariousConnectionNumberTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteVariousConnectionNumberTest extends GridCommonAbstractTest {
+ /** */
+ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int NODES = 6;
+
+ /** */
+ private static Random rnd = new Random();
+
+ /** */
+ private boolean client;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ int connections = rnd.nextInt(10) + 1;
+
+ log.info("Node connections [name=" + gridName + ", connections=" + connections + ']');
+
+ ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setConnectionsPerNode(connections);
+ ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setUsePairedConnections(rnd.nextBoolean());
+ ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setSharedMemoryPort(-1);
+
+ cfg.setClientMode(client);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ long seed = U.currentTimeMillis();
+
+ rnd.setSeed(seed);
+
+ log.info("Random seed: " + seed);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ super.afterTestsStopped();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testVariousConnectionNumber() throws Exception {
+ startGridsMultiThreaded(3);
+
+ client = true;
+
+ startGridsMultiThreaded(3, 3);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setCacheMode(REPLICATED);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ ignite(0).createCache(ccfg);
+
+ for (int i = 0; i < 10; i++) {
+ log.info("Iteration: " + i);
+
+ runOperations(5000);
+
+ awaitPartitionMapExchange();
+
+ int idx = ThreadLocalRandom.current().nextInt(NODES);
+
+ Ignite node = ignite(idx);
+
+ client = node.configuration().isClientMode();
+
+ stopGrid(idx);
+
+ startGrid(idx);
+ }
+ }
+
+ /**
+ * @param time Execution time.
+ * @throws Exception If failed.
+ */
+ private void runOperations(final long time) throws Exception {
+ final AtomicInteger idx = new AtomicInteger();
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Ignite node = ignite(idx.getAndIncrement() % NODES);
+
+ IgniteCache cache = node.cache(null);
+
+ long stopTime = U.currentTimeMillis() + time;
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ while (U.currentTimeMillis() < stopTime) {
+ cache.put(rnd.nextInt(10_000), 0);
+
+ node.compute().broadcast(new DummyJob());
+ }
+
+ return null;
+ }
+ }, NODES * 10, "test-thread");
+ }
+
+ /**
+ *
+ */
+ private static class DummyJob implements IgniteRunnable {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ // No-op.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
index 67ec371..eaa9923 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CrossCacheTxRandomOperationsTest.java
@@ -86,6 +86,11 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 6 * 60 * 1000;
+ }
+
+ /** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
super.beforeTestsStarted();
@@ -170,9 +175,17 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest {
}
/**
+ * @param cacheMode Cache mode.
+ * @param writeSync Write synchronization mode.
+ * @param fairAff Fair affinity flag.
+ * @param ignite Node to use.
+ * @param name Cache name.
*/
- protected void createCache(CacheMode cacheMode, CacheWriteSynchronizationMode writeSync, boolean fairAff,
- Ignite ignite, String name) {
+ protected void createCache(CacheMode cacheMode,
+ CacheWriteSynchronizationMode writeSync,
+ boolean fairAff,
+ Ignite ignite,
+ String name) {
ignite.createCache(cacheConfiguration(name, cacheMode, writeSync, fairAff));
}
@@ -269,9 +282,18 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest {
boolean checkData = fullSync && !optimistic;
+ long stopTime = System.currentTimeMillis() + 10_000;
+
for (int i = 0; i < 10_000; i++) {
- if (i % 100 == 0)
+ if (i % 100 == 0) {
+ if (System.currentTimeMillis() > stopTime) {
+ log.info("Stop on timeout, iteration: " + i);
+
+ break;
+ }
+
log.info("Iteration: " + i);
+ }
boolean rollback = i % 10 == 0;
@@ -557,4 +579,4 @@ public class CrossCacheTxRandomOperationsTest extends GridCommonAbstractTest {
return old;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
index 9405a19..3a2bc81 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java
@@ -200,7 +200,9 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm
private void testRebalance(final Operation operation) throws Exception {
interceptor = new RebalanceUpdateInterceptor();
- for (int iter = 0; iter < TEST_ITERATIONS; iter++) {
+ long stopTime = System.currentTimeMillis() + 2 * 60_000;
+
+ for (int iter = 0; iter < TEST_ITERATIONS && System.currentTimeMillis() < stopTime; iter++) {
log.info("Iteration: " + iter);
failed = false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
index 9458a63..6e2e91f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheOffHeapMultiThreadedUpdateSelfTest.java
@@ -115,10 +115,10 @@ public class GridCacheOffHeapMultiThreadedUpdateSelfTest extends GridCacheOffHea
if (gridCount() > 1)
testPutTx(keyForNode(1), PESSIMISTIC);
}
-
+
/**
* TODO: IGNITE-592.
- *
+ *
* @throws Exception If failed.
*/
public void testPutTxOptimistic() throws Exception {
@@ -227,4 +227,4 @@ public class GridCacheOffHeapMultiThreadedUpdateSelfTest extends GridCacheOffHea
assertFalse(failed);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java
new file mode 100644
index 0000000..30fc9ef
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecovery10ConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+/**
+ *
+ */
+public class IgniteCacheAtomicMessageRecovery10ConnectionsTest extends IgniteCacheAtomicMessageRecoveryTest {
+ /** {@inheritDoc} */
+ @Override protected int connectionsPerNode() {
+ return 10;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
new file mode 100644
index 0000000..71772ef
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+
+/**
+ *
+ */
+public class IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest extends IgniteCacheAtomicMessageRecoveryTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpCommunicationSpi commSpi = (TcpCommunicationSpi)cfg.getCommunicationSpi();
+
+ assertTrue(commSpi.isUsePairedConnections());
+
+ commSpi.setUsePairedConnections(false);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return ATOMIC;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java
new file mode 100644
index 0000000..919aea6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheConnectionRecovery10ConnectionsTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+/**
+ *
+ */
+public class IgniteCacheConnectionRecovery10ConnectionsTest extends IgniteCacheConnectionRecoveryTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setConnectionsPerNode(10);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
index 2f700f3..a91de67 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheCreatePutTest.java
@@ -107,7 +107,7 @@ public class IgniteCacheCreatePutTest extends GridCommonAbstractTest {
try {
int iter = 0;
- while (System.currentTimeMillis() < stopTime) {
+ while (System.currentTimeMillis() < stopTime && iter < 5) {
log.info("Iteration: " + iter++);
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
index 0460a8f..1bfd727 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageRecoveryAbstractTest.java
@@ -58,6 +58,7 @@ public abstract class IgniteCacheMessageRecoveryAbstractTest extends GridCommonA
commSpi.setSocketWriteTimeout(1000);
commSpi.setSharedMemoryPort(-1);
+ commSpi.setConnectionsPerNode(connectionsPerNode());
cfg.setCommunicationSpi(commSpi);
@@ -76,6 +77,13 @@ public abstract class IgniteCacheMessageRecoveryAbstractTest extends GridCommonA
}
/**
+ * @return Value for {@link TcpCommunicationSpi#setConnectionsPerNode(int)}.
+ */
+ protected int connectionsPerNode() {
+ return TcpCommunicationSpi.DFLT_CONN_PER_NODE;
+ }
+
+ /**
* @return Cache atomicity mode.
*/
protected abstract CacheAtomicityMode atomicityMode();
@@ -174,18 +182,22 @@ public abstract class IgniteCacheMessageRecoveryAbstractTest extends GridCommonA
static boolean closeSessions(Ignite ignite) throws Exception {
TcpCommunicationSpi commSpi = (TcpCommunicationSpi)ignite.configuration().getCommunicationSpi();
- Map<UUID, GridCommunicationClient> clients = U.field(commSpi, "clients");
+ Map<UUID, GridCommunicationClient[]> clients = U.field(commSpi, "clients");
boolean closed = false;
- for (GridCommunicationClient client : clients.values()) {
- GridTcpNioCommunicationClient client0 = (GridTcpNioCommunicationClient)client;
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null) {
+ GridTcpNioCommunicationClient client0 = (GridTcpNioCommunicationClient)client;
- GridNioSession ses = client0.session();
+ GridNioSession ses = client0.session();
- ses.close();
+ ses.close();
- closed = true;
+ closed = true;
+ }
+ }
}
return closed;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
index 6256225..0dd4079 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheMessageWriteTimeoutTest.java
@@ -50,8 +50,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
// Try provoke connection close on socket writeTimeout.
commSpi.setSharedMemoryPort(-1);
commSpi.setMessageQueueLimit(10);
- commSpi.setSocketReceiveBuffer(32);
- commSpi.setSocketSendBuffer(32);
+ commSpi.setSocketReceiveBuffer(40);
+ commSpi.setSocketSendBuffer(40);
commSpi.setSocketWriteTimeout(100);
commSpi.setUnacknowledgedMessagesBufferSize(1000);
commSpi.setConnectTimeout(10_000);
@@ -66,15 +66,20 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
super.afterTest();
}
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 10 * 60_000;
+ }
+
/**
* @throws Exception If failed.
*/
public void testMessageQueueLimit() throws Exception {
- startGridsMultiThreaded(3);
-
- for (int i = 0; i < 15; i++) {
+ for (int i = 0; i < 3; i++) {
log.info("Iteration: " + i);
+ startGridsMultiThreaded(3);
+
IgniteInternalFuture<?> fut1 = startJobThreads(50);
U.sleep(100);
@@ -83,6 +88,8 @@ public class IgniteCacheMessageWriteTimeoutTest extends GridCommonAbstractTest {
fut1.get();
fut2.get();
+
+ stopAllGrids();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
index 3fca826..322690c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheMultiTxLockSelfTest.java
@@ -86,7 +86,6 @@ public class IgniteCacheMultiTxLockSelfTest extends GridCommonAbstractTest {
plc.setMaxSize(100000);
ccfg.setEvictionPolicy(plc);
- ccfg.setEvictSynchronized(true);
c.setCacheConfiguration(ccfg);
@@ -95,6 +94,11 @@ public class IgniteCacheMultiTxLockSelfTest extends GridCommonAbstractTest {
return c;
}
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 60_000;
+ }
+
/**
* @throws Exception If failed.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java
new file mode 100644
index 0000000..e8175e5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest extends
+ GridCacheAtomicPrimaryWriteOrderMultiNodeFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setStripedPoolSize(-1);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java
new file mode 100644
index 0000000..05fe85f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest extends
+ GridCachePartitionedMultiNodeFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setStripedPoolSize(-1);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
index c9d18eb..e9d74ff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDeadlockDetectionNoHangsTest.java
@@ -211,7 +211,7 @@ public class TxDeadlockDetectionNoHangsTest extends GridCommonAbstractTest {
tx.commit();
}
catch (Exception e) {
- e.printStackTrace();
+ log.info("Ignore error: " + e);
}
}
}, NODES_CNT * 3, "tx-thread");
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
index aa240aa..f6a06c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionTest.java
@@ -111,6 +111,9 @@ public class TxOptimisticDeadlockDetectionTest extends GridCommonAbstractTest {
cfg.setClientMode(client);
+ // Test spi blocks message send, this can cause hang with striped pool.
+ cfg.setStripedPoolSize(-1);
+
return cfg;
}
@@ -274,8 +277,8 @@ public class TxOptimisticDeadlockDetectionTest extends GridCommonAbstractTest {
Object k;
- log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
- ", tx=" + tx + ", key=" + transformer.apply(key) + ']');
+ log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode().id() +
+ ", tx=" + tx.xid() + ", key=" + transformer.apply(key) + ']');
cache.put(transformer.apply(key), 0);
@@ -309,23 +312,27 @@ public class TxOptimisticDeadlockDetectionTest extends GridCommonAbstractTest {
entries.put(k, 2);
}
- log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
- ", tx=" + tx + ", entries=" + entries + ']');
+ log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode().id() +
+ ", tx=" + tx.xid() + ", entries=" + entries + ']');
cache.putAll(entries);
tx.commit();
}
catch (Throwable e) {
- U.error(log, "Expected exception: ", e);
+ log.info("Expected exception: " + e);
+
+ e.printStackTrace(System.out);
// At least one stack trace should contain TransactionDeadlockException.
if (hasCause(e, TransactionTimeoutException.class) &&
- hasCause(e, TransactionDeadlockException.class)
- ) {
- if (deadlockErr.compareAndSet(null, cause(e, TransactionDeadlockException.class)))
- U.error(log, "At least one stack trace should contain " +
- TransactionDeadlockException.class.getSimpleName(), e);
+ hasCause(e, TransactionDeadlockException.class)) {
+ if (deadlockErr.compareAndSet(null, cause(e, TransactionDeadlockException.class))) {
+ log.info("At least one stack trace should contain " +
+ TransactionDeadlockException.class.getSimpleName());
+
+ e.printStackTrace(System.out);
+ }
}
}
}
@@ -344,7 +351,7 @@ public class TxOptimisticDeadlockDetectionTest extends GridCommonAbstractTest {
TransactionDeadlockException deadlockE = deadlockErr.get();
- assertNotNull(deadlockE);
+ assertNotNull("Failed to detect deadlock", deadlockE);
boolean fail = false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index 6fc7e02..7b5abf5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -372,4 +372,4 @@ public class GridServiceProcessorProxySelfTest extends GridServiceProcessorAbstr
X.println("Executing cache service: " + ctx.name());
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
index adcd144..4bc9f01 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/future/GridFutureAdapterSelfTest.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.util.future;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -227,87 +228,98 @@ public class GridFutureAdapterSelfTest extends GridCommonAbstractTest {
*
* @throws Exception In case of any exception.
*/
- @SuppressWarnings("ErrorNotRethrown")
public void testChaining() throws Exception {
+ checkChaining(null);
+
+ ExecutorService exec = Executors.newFixedThreadPool(1);
+
+ try {
+ checkChaining(exec);
+
+ GridFinishedFuture<Integer> fut = new GridFinishedFuture<>(1);
+
+ IgniteInternalFuture<Object> chain = fut.chain(new CX1<IgniteInternalFuture<Integer>, Object>() {
+ @Override public Object applyx(IgniteInternalFuture<Integer> fut) throws IgniteCheckedException {
+ return fut.get() + 1;
+ }
+ }, exec);
+
+ assertEquals(2, chain.get());
+ }
+ finally {
+ exec.shutdown();
+ }
+ }
+
+ /**
+ * @param exec Executor for chain callback.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ErrorNotRethrown")
+ private void checkChaining(ExecutorService exec) throws Exception {
final CX1<IgniteInternalFuture<Object>, Object> passThrough = new CX1<IgniteInternalFuture<Object>, Object>() {
@Override public Object applyx(IgniteInternalFuture<Object> f) throws IgniteCheckedException {
return f.get();
}
};
- final GridTestKernalContext ctx = new GridTestKernalContext(log);
-
- ctx.setExecutorService(Executors.newFixedThreadPool(1));
- ctx.setSystemExecutorService(Executors.newFixedThreadPool(1));
-
- ctx.add(new PoolProcessor(ctx));
- ctx.add(new GridClosureProcessor(ctx));
+ GridFutureAdapter<Object> fut = new GridFutureAdapter<>();
+ IgniteInternalFuture<Object> chain = exec != null ? fut.chain(passThrough, exec) : fut.chain(passThrough);
- ctx.start();
+ assertFalse(fut.isDone());
+ assertFalse(chain.isDone());
try {
- // Test result returned.
-
- GridFutureAdapter<Object> fut = new GridFutureAdapter<>();
- IgniteInternalFuture<Object> chain = fut.chain(passThrough);
+ chain.get(20);
- assertFalse(fut.isDone());
- assertFalse(chain.isDone());
-
- try {
- chain.get(20);
-
- fail("Expects timeout exception.");
- }
- catch (IgniteFutureTimeoutCheckedException e) {
- info("Expected timeout exception: " + e.getMessage());
- }
+ fail("Expects timeout exception.");
+ }
+ catch (IgniteFutureTimeoutCheckedException e) {
+ info("Expected timeout exception: " + e.getMessage());
+ }
- fut.onDone("result");
+ fut.onDone("result");
- assertEquals("result", chain.get(1));
+ assertEquals("result", chain.get(1));
- // Test exception re-thrown.
+ // Test exception re-thrown.
- fut = new GridFutureAdapter<>();
- chain = fut.chain(passThrough);
+ fut = new GridFutureAdapter<>();
+ chain = exec != null ? fut.chain(passThrough, exec) : fut.chain(passThrough);
- fut.onDone(new ClusterGroupEmptyCheckedException("test exception"));
+ fut.onDone(new ClusterGroupEmptyCheckedException("test exception"));
- try {
- chain.get();
+ try {
+ chain.get();
- fail("Expects failed with exception.");
- }
- catch (ClusterGroupEmptyCheckedException e) {
- info("Expected exception: " + e.getMessage());
- }
+ fail("Expects failed with exception.");
+ }
+ catch (ClusterGroupEmptyCheckedException e) {
+ info("Expected exception: " + e.getMessage());
+ }
- // Test error re-thrown.
+ // Test error re-thrown.
- fut = new GridFutureAdapter<>();
- chain = fut.chain(passThrough);
+ fut = new GridFutureAdapter<>();
+ chain = exec != null ? fut.chain(passThrough, exec) : fut.chain(passThrough);
- try {
- fut.onDone(new StackOverflowError("test error"));
+ try {
+ fut.onDone(new StackOverflowError("test error"));
+ if (exec == null)
fail("Expects failed with error.");
- }
- catch (StackOverflowError e) {
- info("Expected error: " + e.getMessage());
- }
+ }
+ catch (StackOverflowError e) {
+ info("Expected error: " + e.getMessage());
+ }
- try {
- chain.get();
+ try {
+ chain.get();
- fail("Expects failed with error.");
- }
- catch (StackOverflowError e) {
- info("Expected error: " + e.getMessage());
- }
+ fail("Expects failed with error.");
}
- finally {
- ctx.stop(false);
+ catch (StackOverflowError e) {
+ info("Expected error: " + e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
index 201fd27..d403784 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/nio/impl/GridNioFilterChainSelfTest.java
@@ -114,7 +114,7 @@ public class GridNioFilterChainSelfTest extends GridCommonAbstractTest {
proceedExceptionCaught(ses, ex);
}
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) {
sndEvt.compareAndSet(null, ses.<String>meta(MESSAGE_WRITE_META_NAME));
sndMsgObj.compareAndSet(null, msg);
@@ -155,7 +155,7 @@ public class GridNioFilterChainSelfTest extends GridCommonAbstractTest {
chain.onSessionIdleTimeout(ses);
chain.onSessionWriteTimeout(ses);
assertNull(chain.onSessionClose(ses));
- assertNull(chain.onSessionWrite(ses, snd));
+ assertNull(chain.onSessionWrite(ses, snd, true));
assertEquals("DCBA", connectedEvt.get());
assertEquals("DCBA", disconnectedEvt.get());
@@ -210,10 +210,10 @@ public class GridNioFilterChainSelfTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
chainMeta(ses, MESSAGE_WRITE_META_NAME);
- return proceedSessionWrite(ses, msg);
+ return proceedSessionWrite(ses, msg, fut);
}
/** {@inheritDoc} */
@@ -349,6 +349,11 @@ public class GridNioFilterChainSelfTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
+ @Override public void sendNoFuture(Object msg) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public GridNioFuture<Object> resumeReads() {
return null;
}
@@ -369,13 +374,28 @@ public class GridNioFilterChainSelfTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
- @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
// No-op.
}
/** {@inheritDoc} */
- @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
+ @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
return null;
}
+
+ /** {@inheritDoc} */
+ @Override public void systemMessage(Object msg) {
+ // No-op.
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java b/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
index 61a13b1..25dd780 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/nio/GridNioBenchmarkClient.java
@@ -83,7 +83,7 @@ public class GridNioBenchmarkClient {
*/
public void run() throws IOException, InterruptedException {
for (int i = 0; i < connCnt; i++)
- exec.submit(new ClientThread());
+ exec.execute(new ClientThread());
Thread.sleep(5*60*1000);
@@ -167,4 +167,4 @@ public class GridNioBenchmarkClient {
return read;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
index f21f31b..a18ef32 100644
--- a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PRecursionTaskSelfTest.java
@@ -196,4 +196,4 @@ public class GridP2PRecursionTaskSelfTest extends GridCommonAbstractTest {
return ignite.compute().execute(FactorialTask.class, arg);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
index 652e47f..5ca8f26 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/GridTcpSpiForwardingSelfTest.java
@@ -69,7 +69,7 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
private static final int commExtPort2 = 20100;
/** */
- private AddressResolver resolver;
+ private AddressResolver rslvr;
/** */
private boolean ipFinderUseLocPorts;
@@ -111,14 +111,15 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
cfg.setConnectorConfiguration(null);
TcpCommunicationSpi commSpi = new TcpCommunicationSpi() {
- @Override protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+ @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
+ throws IgniteCheckedException {
Map<String, Object> attrs = new HashMap<>(node.attributes());
attrs.remove(createSpiAttributeName(ATTR_PORT));
((TcpDiscoveryNode)node).setAttributes(attrs);
- return super.createTcpClient(node);
+ return super.createTcpClient(node, connIdx);
}
};
@@ -126,12 +127,13 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
commSpi.setLocalPort(commLocPort);
commSpi.setLocalPortRange(1);
commSpi.setSharedMemoryPort(-1);
+ commSpi.setConnectionsPerNode(1);
cfg.setCommunicationSpi(commSpi);
- assert resolver != null;
+ assert rslvr != null;
- cfg.setAddressResolver(resolver);
+ cfg.setAddressResolver(rslvr);
return cfg;
}
@@ -147,7 +149,7 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
map.put(new InetSocketAddress("127.0.0.1", locPort2), F.asList(new InetSocketAddress("127.0.0.1", extPort2)));
map.put(new InetSocketAddress("127.0.0.1", commLocPort2), F.asList(new InetSocketAddress("127.0.0.1", commExtPort2)));
- resolver = new AddressResolver() {
+ rslvr = new AddressResolver() {
@Override public Collection<InetSocketAddress> getExternalAddresses(InetSocketAddress addr) {
return map.get(addr);
}
@@ -167,7 +169,7 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
map.put("127.0.0.1:" + locPort2, "127.0.0.1:" + extPort2);
map.put("127.0.0.1:" + commLocPort2, "127.0.0.1:" + commExtPort2);
- resolver = new BasicAddressResolver(map);
+ rslvr = new BasicAddressResolver(map);
doTestForward();
}
@@ -180,7 +182,7 @@ public class GridTcpSpiForwardingSelfTest extends GridCommonAbstractTest {
map.put("127.0.0.1", "127.0.0.1");
- resolver = new BasicAddressResolver(map);
+ rslvr = new BasicAddressResolver(map);
ipFinderUseLocPorts = true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
index 076724d..3c4fea0 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
@@ -90,16 +90,36 @@ abstract class GridTcpCommunicationSpiAbstractTest extends GridAbstractCommunica
super.afterTest();
for (CommunicationSpi spi : spis.values()) {
- ConcurrentMap<UUID, GridCommunicationClient> clients = U.field(spi, "clients");
+ ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(spi, "clients");
+
+ for (int i = 0; i < 20; i++) {
+ GridCommunicationClient client0 = null;
+
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null) {
+ client0 = client;
+
+ break;
+ }
+ }
+
+ if (client0 != null)
+ break;
+ }
+
+ if (client0 == null)
+ return;
- for (int i = 0; i < 20 && !clients.isEmpty(); i++) {
info("Check failed for SPI [grid=" +
- GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "gridName") + ", spi=" + spi + ']');
+ GridTestUtils.getFieldValue(spi, IgniteSpiAdapter.class, "gridName") +
+ ", client=" + client0 +
+ ", spi=" + spi + ']');
U.sleep(1000);
}
- assert clients.isEmpty() : "Clients: " + clients;
+ fail("Failed to wait when clients are closed.");
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
index 8635d94..a649130 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConcurrentConnectSelfTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -83,6 +84,12 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
/** Use ssl. */
protected boolean useSsl;
+ /** */
+ private int connectionsPerNode = 1;
+
+ /** */
+ private boolean pairedConnections = true;
+
/**
*
*/
@@ -163,6 +170,34 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
/**
* @throws Exception If failed.
*/
+ public void testMultithreaded_10Connections() throws Exception {
+ connectionsPerNode = 10;
+
+ testMultithreaded();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMultithreaded_NoPairedConnections() throws Exception {
+ pairedConnections = false;
+
+ testMultithreaded();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMultithreaded_10ConnectionsNoPaired() throws Exception {
+ pairedConnections = false;
+ connectionsPerNode = 10;
+
+ testMultithreaded();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testWithLoad() throws Exception {
int threads = Runtime.getRuntime().availableProcessors() * 5;
@@ -244,7 +279,7 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
final AtomicInteger idx = new AtomicInteger();
try {
- GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ final Callable<Void> c = new Callable<Void>() {
@Override public Void call() throws Exception {
int idx0 = idx.getAndIncrement();
@@ -270,7 +305,40 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
return null;
}
- }, threads, "test");
+ };
+
+ List<Thread> threadsList = new ArrayList<>();
+
+ final AtomicBoolean fail = new AtomicBoolean();
+
+ final AtomicLong tId = new AtomicLong();
+
+ for (int t = 0; t < threads; t++) {
+ Thread t0 = new Thread(new Runnable() {
+ @Override public void run() {
+ try {
+ c.call();
+ }
+ catch (Throwable e) {
+ log.error("Unexpected error: " + e, e);
+
+ fail.set(true);
+ }
+ }
+ }) {
+ @Override public long getId() {
+ // Override getId to use all connections.
+ return tId.getAndIncrement();
+ }
+ };
+
+ threadsList.add(t0);
+
+ t0.start();
+ }
+
+ for (Thread t0 : threadsList)
+ t0.join();
assertTrue(latch.await(10, TimeUnit.SECONDS));
@@ -281,17 +349,19 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
final GridNioServer srv = U.field(spi, "nioSrvr");
+ final int conns = pairedConnections ? 2 : 1;
+
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
Collection sessions = U.field(srv, "sessions");
- return sessions.size() == 1;
+ return sessions.size() == conns * connectionsPerNode;
}
}, 5000);
Collection sessions = U.field(srv, "sessions");
- assertEquals(1, sessions.size());
+ assertEquals(conns * connectionsPerNode, sessions.size());
}
assertEquals(expMsgs, lsnr.cntr.get());
@@ -320,6 +390,8 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
spi.setIdleConnectionTimeout(60_000);
spi.setConnectTimeout(10_000);
spi.setSharedMemoryPort(-1);
+ spi.setConnectionsPerNode(connectionsPerNode);
+ spi.setUsePairedConnections(pairedConnections);
return spi;
}
@@ -434,4 +506,4 @@ public class GridTcpCommunicationSpiConcurrentConnectSelfTest<T extends Communic
rsrcs.stopThreads();
}
-}
\ No newline at end of file
+}
[16/50] [abbrv] ignite git commit: IGNITE-4276: Hadoop: added
configurable throttle for shuffle message sending. Disabled by default.
Posted by sb...@apache.org.
IGNITE-4276: Hadoop: added configurable throttle for shuffle message sending. Disabled by default.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ae903c59
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ae903c59
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ae903c59
Branch: refs/heads/master
Commit: ae903c59c10bb9fad58842221f3e720d1b4d54b8
Parents: bf1770b
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 5 14:57:25 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:44:58 2016 +0300
----------------------------------------------------------------------
.../internal/processors/hadoop/HadoopJobProperty.java | 10 +++++++++-
.../processors/hadoop/shuffle/HadoopShuffleJob.java | 9 ++++++++-
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae903c59/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index dcfbcba..e7bf565 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -81,7 +81,15 @@ public enum HadoopJobProperty {
* <p>
* By default is {@code false}.
*/
- SHUFFLE_REDUCER_NO_SORTING;
+ SHUFFLE_REDUCER_NO_SORTING,
+
+ /**
+ * Shuffle job throttle in milliseconds. When job is executed with separate shuffle thread, this parameter
+ * controls sleep duration between iterations through intermediate reducer maps.
+ * <p>
+ * Defaults to {@code 0}.
+ */
+ SHUFFLE_JOB_THROTTLE;
/** */
private final String ptyName;
http://git-wip-us.apache.org/repos/asf/ignite/blob/ae903c59/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index b940c72..8c731c0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -55,6 +55,7 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.thread.IgniteThread;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -108,6 +109,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/** */
private final IgniteLogger log;
+ /** */
+ private final long throttle;
+
/**
* @param locReduceAddr Local reducer address.
* @param log Logger.
@@ -136,6 +140,8 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
maps = new AtomicReferenceArray<>(totalReducerCnt);
msgs = new HadoopShuffleMessage[totalReducerCnt];
+
+ throttle = get(job.info(), SHUFFLE_JOB_THROTTLE, 0);
}
/**
@@ -175,7 +181,8 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
@Override protected void body() throws InterruptedException {
try {
while (!isCancelled()) {
- Thread.sleep(5);
+ if (throttle > 0)
+ Thread.sleep(throttle);
collectUpdatesAndSend(false);
}
[23/50] [abbrv] ignite git commit: IGNITE-4301: Hadoop: Optimized
shuffle so that only one ack is needed when running in embedded mode. This
closes #1319.
Posted by sb...@apache.org.
IGNITE-4301: Hadoop: Optimized shuffle so that only one ack is needed when running in embedded mode. This closes #1319.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/58c33805
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/58c33805
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/58c33805
Branch: refs/heads/master
Commit: 58c338051aed89a787c2a575856e949cc44e8b8c
Parents: e857f29
Author: devozerov <vo...@gridgain.com>
Authored: Tue Dec 6 13:23:05 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:14 2016 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 12 +
.../shuffle/HadoopShuffleFinishRequest.java | 172 ++++++++++++++
.../shuffle/HadoopShuffleFinishResponse.java | 142 ++++++++++++
.../hadoop/shuffle/HadoopShuffle.java | 45 ++--
.../hadoop/shuffle/HadoopShuffleJob.java | 231 +++++++++++++++++--
.../hadoop/shuffle/HadoopShuffleLocalState.java | 67 ++++++
.../shuffle/HadoopShuffleRemoteState.java | 64 +++++
.../child/HadoopChildProcessRunner.java | 6 +-
8 files changed, 686 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 86742e8..4ffb220 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -119,6 +119,8 @@ import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerResponse;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse;
import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage;
import org.apache.ignite.internal.processors.igfs.IgfsAckMessage;
import org.apache.ignite.internal.processors.igfs.IgfsBlockKey;
@@ -168,6 +170,16 @@ public class GridIoMessageFactory implements MessageFactory {
Message msg = null;
switch (type) {
+ case -41:
+ msg = new HadoopShuffleFinishResponse();
+
+ break;
+
+ case -40:
+ msg = new HadoopShuffleFinishRequest();
+
+ break;
+
case -39:
msg = new HadoopJobId();
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java
new file mode 100644
index 0000000..f568c2e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishRequest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Shuffle finish request.
+ */
+public class HadoopShuffleFinishRequest implements Message, HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** Total message count. */
+ private long msgCnt;
+
+ /**
+ * Default constructor.
+ */
+ public HadoopShuffleFinishRequest() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param jobId Job.
+ * @param msgCnt Message count.
+ */
+ public HadoopShuffleFinishRequest(HadoopJobId jobId, long msgCnt) {
+ this.jobId = jobId;
+ this.msgCnt = msgCnt;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Message count.
+ */
+ public long messageCount() {
+ return msgCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeMessage("jobId", jobId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeLong("msgCnt", msgCnt))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ jobId = reader.readMessage("jobId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ msgCnt = reader.readLong("msgCnt");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(HadoopShuffleFinishRequest.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -40;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+
+ out.writeLong(msgCnt);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+
+ jobId.readExternal(in);
+
+ msgCnt = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopShuffleFinishRequest.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java
new file mode 100644
index 0000000..4b7c93b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleFinishResponse.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Shuffle finish response.
+ */
+public class HadoopShuffleFinishResponse implements Message, HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Job ID. */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /**
+ * Default constructor.
+ */
+ public HadoopShuffleFinishResponse() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param jobId Job.
+ */
+ public HadoopShuffleFinishResponse(HadoopJobId jobId) {
+ this.jobId = jobId;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeMessage("jobId", jobId))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ jobId = reader.readMessage("jobId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(HadoopShuffleFinishResponse.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -41;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+
+ jobId.readExternal(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopShuffleFinishResponse.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 4450bf2..82bbd32 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -103,7 +103,7 @@ public class HadoopShuffle extends HadoopComponent {
HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log,
- ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()));
+ ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), true);
UUID[] rdcAddrs = new UUID[plan.reducers()];
@@ -189,37 +189,34 @@ public class HadoopShuffle extends HadoopComponent {
* @return {@code True}.
*/
public boolean onMessageReceived(UUID src, HadoopMessage msg) {
- if (msg instanceof HadoopShuffleMessage) {
- HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
+ try {
+ if (msg instanceof HadoopShuffleMessage) {
+ HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
- try {
- job(m.jobId()).onShuffleMessage(m);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Message handling failed.", e);
+ job(m.jobId()).onShuffleMessage(src, m);
}
+ else if (msg instanceof HadoopShuffleAck) {
+ HadoopShuffleAck m = (HadoopShuffleAck)msg;
- try {
- // Reply with ack.
- send0(src, new HadoopShuffleAck(m.id(), m.jobId()));
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']', e);
+ job(m.jobId()).onShuffleAck(m);
}
- }
- else if (msg instanceof HadoopShuffleAck) {
- HadoopShuffleAck m = (HadoopShuffleAck)msg;
+ else if (msg instanceof HadoopShuffleFinishRequest) {
+ HadoopShuffleFinishRequest m = (HadoopShuffleFinishRequest)msg;
- try {
- job(m.jobId()).onShuffleAck(m);
+ job(m.jobId()).onShuffleFinishRequest(src, m);
}
- catch (IgniteCheckedException e) {
- U.error(log, "Message handling failed.", e);
+ else if (msg instanceof HadoopShuffleFinishResponse) {
+ HadoopShuffleFinishResponse m = (HadoopShuffleFinishResponse)msg;
+
+ job(m.jobId()).onShuffleFinishResponse(src);
}
+ else
+ throw new IllegalStateException("Unknown message type received to Hadoop shuffle [src=" + src +
+ ", msg=" + msg + ']');
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Message handling failed.", e);
}
- else
- throw new IllegalStateException("Unknown message type received to Hadoop shuffle [src=" + src +
- ", msg=" + msg + ']');
return true;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 3afb55a..0a3a0ae 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -17,7 +17,9 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@@ -27,6 +29,7 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
@@ -118,9 +121,21 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/** Message size. */
private final int msgSize;
+ /** Local shuffle states. */
+ private volatile HashMap<T, HadoopShuffleLocalState> locShuffleStates = new HashMap<>();
+
+ /** Remote shuffle states. */
+ private volatile HashMap<T, HadoopShuffleRemoteState> rmtShuffleStates = new HashMap<>();
+
+ /** Mutex for internal synchronization. */
+ private final Object mux = new Object();
+
/** */
private final long throttle;
+ /** Embedded mode flag. */
+ private final boolean embedded;
+
/**
* @param locReduceAddr Local reducer address.
* @param log Logger.
@@ -128,15 +143,17 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
* @param mem Memory.
* @param totalReducerCnt Amount of reducers in the Job.
* @param locReducers Reducers will work on current node.
+ * @param embedded Whether shuffle is running in embedded mode.
* @throws IgniteCheckedException If error.
*/
public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
- int totalReducerCnt, int[] locReducers) throws IgniteCheckedException {
+ int totalReducerCnt, int[] locReducers, boolean embedded) throws IgniteCheckedException {
this.locReduceAddr = locReduceAddr;
this.totalReducerCnt = totalReducerCnt;
this.job = job;
this.mem = mem;
this.log = log.getLogger(HadoopShuffleJob.class);
+ this.embedded = embedded;
msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
@@ -238,10 +255,11 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
/**
+ * @param src Source.
* @param msg Message.
* @throws IgniteCheckedException Exception.
*/
- public void onShuffleMessage(HadoopShuffleMessage msg) throws IgniteCheckedException {
+ public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheckedException {
assert msg.buffer() != null;
assert msg.offset() > 0;
@@ -276,6 +294,15 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
});
}
+
+ if (embedded) {
+ // No immediate response.
+ if (localShuffleState(src).onShuffleMessage())
+ sendFinishResponse(src, msg.jobId());
+ }
+ else
+ // Response for every message.
+ io.apply(src, new HadoopShuffleAck(msg.id(), msg.jobId()));
}
/**
@@ -292,6 +319,121 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
/**
+ * Process shuffle finish request.
+ *
+ * @param src Source.
+ * @param msg Shuffle finish message.
+ */
+ public void onShuffleFinishRequest(T src, HadoopShuffleFinishRequest msg) {
+ if (log.isDebugEnabled())
+ log.debug("Received shuffle finish request [jobId=" + job.id() + ", src=" + src + ", req=" + msg + ']');
+
+ HadoopShuffleLocalState state = localShuffleState(src);
+
+ if (state.onShuffleFinishMessage(msg.messageCount()))
+ sendFinishResponse(src, msg.jobId());
+ }
+
+ /**
+ * Process shuffle finish response.
+ *
+ * @param src Source.
+ */
+ public void onShuffleFinishResponse(T src) {
+ if (log.isDebugEnabled())
+ log.debug("Received shuffle finish response [jobId=" + job.id() + ", src=" + src + ']');
+
+ remoteShuffleState(src).onShuffleFinishResponse();
+ }
+
+ /**
+ * Send finish response.
+ *
+ * @param dest Destination.
+ * @param jobId Job ID.
+ */
+ @SuppressWarnings("unchecked")
+ private void sendFinishResponse(T dest, HadoopJobId jobId) {
+ if (log.isDebugEnabled())
+ log.debug("Sent shuffle finish response [jobId=" + jobId + ", dest=" + dest + ']');
+
+ HadoopShuffleFinishResponse msg = new HadoopShuffleFinishResponse(jobId);
+
+ io.apply(dest, msg);
+ }
+
+ /**
+ * Get local shuffle state for node.
+ *
+ * @param src Source
+ * @return Local shuffle state.
+ */
+ private HadoopShuffleLocalState localShuffleState(T src) {
+ HashMap<T, HadoopShuffleLocalState> states = locShuffleStates;
+
+ HadoopShuffleLocalState res = states.get(src);
+
+ if (res == null) {
+ synchronized (mux) {
+ res = locShuffleStates.get(src);
+
+ if (res == null) {
+ res = new HadoopShuffleLocalState();
+
+ states = new HashMap<>(locShuffleStates);
+
+ states.put(src, res);
+
+ locShuffleStates = states;
+ }
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Get remote shuffle state for node.
+ *
+ * @param src Source.
+ * @return Remote shuffle state.
+ */
+ private HadoopShuffleRemoteState remoteShuffleState(T src) {
+ HashMap<T, HadoopShuffleRemoteState> states = rmtShuffleStates;
+
+ HadoopShuffleRemoteState res = states.get(src);
+
+ if (res == null) {
+ synchronized (mux) {
+ res = rmtShuffleStates.get(src);
+
+ if (res == null) {
+ res = new HadoopShuffleRemoteState();
+
+ states = new HashMap<>(rmtShuffleStates);
+
+ states.put(src, res);
+
+ rmtShuffleStates = states;
+ }
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Get all remote shuffle states.
+ *
+ * @return Remote shuffle states.
+ */
+ private HashMap<T, HadoopShuffleRemoteState> remoteShuffleStates() {
+ synchronized (mux) {
+ return new HashMap<>(rmtShuffleStates);
+ }
+ }
+
+ /**
* Unsafe value.
*/
private static class UnsafeValue implements HadoopMultimap.Value {
@@ -406,38 +548,50 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
* @param newBufMinSize Min new buffer size.
*/
private void send(final int idx, int newBufMinSize) {
- final GridFutureAdapter<?> fut = new GridFutureAdapter<>();
-
HadoopShuffleMessage msg = msgs[idx];
final long msgId = msg.id();
- IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> old = sentMsgs.putIfAbsent(msgId,
- new IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>>(msg, fut));
+ final GridFutureAdapter<?> fut;
+
+ if (embedded)
+ fut = null;
+ else {
+ fut = new GridFutureAdapter<>();
+
+ IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> old = sentMsgs.putIfAbsent(msgId,
+ new IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>>(msg, fut));
- assert old == null;
+ assert old == null;
+ }
try {
io.apply(reduceAddrs[idx], msg);
+
+ if (embedded)
+ remoteShuffleState(reduceAddrs[idx]).onShuffleMessage();
}
catch (GridClosureException e) {
- fut.onDone(U.unwrap(e));
+ if (fut != null)
+ fut.onDone(U.unwrap(e));
}
- fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> f) {
- try {
- f.get();
+ if (fut != null) {
+ fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> f) {
+ try {
+ f.get();
- // Clean up the future from map only if there was no exception.
- // Otherwise flush() should fail.
- sentMsgs.remove(msgId);
- }
- catch (IgniteCheckedException e) {
- log.error("Failed to send message.", e);
+ // Clean up the future from map only if there was no exception.
+ // Otherwise flush() should fail.
+ sentMsgs.remove(msgId);
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to send message.", e);
+ }
}
- }
- });
+ });
+ }
msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
Math.max(msgSize, newBufMinSize));
@@ -513,14 +667,41 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
GridCompoundFuture fut = new GridCompoundFuture<>();
- for (IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> tup : sentMsgs.values())
- fut.add(tup.get2());
+ if (embedded) {
+ boolean sent = false;
- fut.markInitialized();
+ for (Map.Entry<T, HadoopShuffleRemoteState> rmtStateEntry : remoteShuffleStates().entrySet()) {
+ T dest = rmtStateEntry.getKey();
+ HadoopShuffleRemoteState rmtState = rmtStateEntry.getValue();
- if (log.isDebugEnabled())
- log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
+ HadoopShuffleFinishRequest req = new HadoopShuffleFinishRequest(job.id(), rmtState.messageCount());
+
+ io.apply(dest, req);
+ if (log.isDebugEnabled())
+ log.debug("Sent shuffle finish request [jobId=" + job.id() + ", dest=" + dest +
+ ", req=" + req + ']');
+
+ fut.add(rmtState.future());
+
+ sent = true;
+ }
+
+ if (sent)
+ fut.markInitialized();
+ else
+ return new GridFinishedFuture<>();
+ }
+ else {
+ for (IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>> tup : sentMsgs.values())
+ fut.add(tup.get2());
+
+ fut.markInitialized();
+
+ if (log.isDebugEnabled())
+ log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
+
+ }
return fut;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java
new file mode 100644
index 0000000..68c0653
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleLocalState.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Local shuffle state.
+ */
+class HadoopShuffleLocalState {
+ /** Message counter. */
+ private final AtomicLong msgCnt = new AtomicLong();
+
+ /** Reply guard. */
+ private final AtomicBoolean replyGuard = new AtomicBoolean();
+
+ /** Total message count.*/
+ private volatile long totalMsgCnt;
+
+ /**
+ * Callback invoked when shuffle message arrived.
+ *
+ * @return Whether to perform reply.
+ */
+ public boolean onShuffleMessage() {
+ long msgCnt0 = msgCnt.incrementAndGet();
+
+ return msgCnt0 == totalMsgCnt && reserve();
+ }
+
+ /**
+ * Callback invoked when shuffle is finished.
+ *
+ * @param totalMsgCnt Message count.
+ * @return Whether to perform reply.
+ */
+ public boolean onShuffleFinishMessage(long totalMsgCnt) {
+ this.totalMsgCnt = totalMsgCnt;
+
+ return msgCnt.get() == totalMsgCnt && reserve();
+ }
+
+ /**
+ * Reserve reply.
+ *
+ * @return {@code True} if reserved.
+ */
+ private boolean reserve() {
+ return replyGuard.compareAndSet(false, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
new file mode 100644
index 0000000..5ffaa55
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.lang.IgniteInClosure;
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Remote shuffle state.
+ */
+class HadoopShuffleRemoteState<T> {
+ /** Message count. */
+ private final AtomicLong msgCnt = new AtomicLong();
+
+ /** Completion future. */
+ private final GridFutureAdapter fut = new GridFutureAdapter();
+
+ /**
+ * Callback invoked when shuffle message is sent.
+ */
+ public void onShuffleMessage() {
+ msgCnt.incrementAndGet();
+ }
+
+ /**
+ * Callback invoked on shuffle finish response.
+ */
+ public void onShuffleFinishResponse() {
+ fut.onDone();
+ }
+
+ /**
+ * @return Message count.
+ */
+ public long messageCount() {
+ return msgCnt.get();
+ }
+
+ /**
+ * @return Completion future.
+ */
+ public GridFutureAdapter future() {
+ return fut;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58c33805/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
index 7001b8c..cb08c00 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -151,7 +151,7 @@ public class HadoopChildProcessRunner {
job.initialize(true, nodeDesc.processId());
shuffleJob = new HadoopShuffleJob<>(comm.localProcessDescriptor(), log, job, mem,
- req.totalReducerCount(), req.localReducers());
+ req.totalReducerCount(), req.localReducers(), false);
initializeExecutors(req);
@@ -432,9 +432,7 @@ public class HadoopChildProcessRunner {
try {
HadoopShuffleMessage m = (HadoopShuffleMessage)msg;
- shuffleJob.onShuffleMessage(m);
-
- comm.sendMessage(desc, new HadoopShuffleAck(m.id(), m.jobId()));
+ shuffleJob.onShuffleMessage(desc, m);
}
catch (IgniteCheckedException e) {
U.error(log, "Failed to process hadoop shuffle message [desc=" + desc + ", msg=" + msg + ']', e);
[30/50] [abbrv] ignite git commit: IGNITE-4277: Hadoop: better
property naming for "partially raw" comparator (2).
Posted by sb...@apache.org.
IGNITE-4277: Hadoop: better property naming for "partially raw" comparator (2).
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/57eb47f9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/57eb47f9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/57eb47f9
Branch: refs/heads/master
Commit: 57eb47f9c057d15739987500a8b3894d1d739d29
Parents: f8ac0f1
Author: devozerov <vo...@gridgain.com>
Authored: Thu Dec 15 12:29:51 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:56 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/hadoop/HadoopClassLoader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/57eb47f9/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 81c1405..1209e74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -372,7 +372,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
// We use "contains" instead of "equals" to handle subclasses properly.
if (clsName.contains("org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem") ||
clsName.contains("org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem") ||
- clsName.contains("org.apache.ignite.hadoop.io.TextPartialRawComparator") ||
+ clsName.contains("org.apache.ignite.hadoop.io.TextPartiallyRawComparator") ||
clsName.contains("org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider"))
return true;
}
[45/50] [abbrv] ignite git commit: Null argument check for
cache.putAll.
Posted by sb...@apache.org.
Null argument check for cache.putAll.
(cherry picked from commit c0f1a21)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e82eefe7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e82eefe7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e82eefe7
Branch: refs/heads/master
Commit: e82eefe7a7d7f541e6d2fc4b166843b87ec9ef87
Parents: 09c634e
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 20 12:35:51 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 20 13:21:28 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/processors/cache/GridCacheAdapter.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e82eefe7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 8187e8f..f235f6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2661,6 +2661,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** {@inheritDoc} */
@Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+ A.notNull(m, "map");
+
if (F.isEmpty(m))
return;
[38/50] [abbrv] ignite git commit: ignite-4411 Changed test to avoid
deadlocks when striped pool is used.
Posted by sb...@apache.org.
ignite-4411 Changed test to avoid deadlocks when striped pool is used.
(cherry picked from commit 9a62d53)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/28dab6ea
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/28dab6ea
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/28dab6ea
Branch: refs/heads/master
Commit: 28dab6ea7573affe95512a403821cee5d82fcaf5
Parents: 05dd08b
Author: sboikov <sb...@gridgain.com>
Authored: Thu Dec 15 12:15:22 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 16 14:11:04 2016 +0300
----------------------------------------------------------------------
...achePartitionedMultiNodeFullApiSelfTest.java | 37 ++++++++++++++++----
1 file changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/28dab6ea/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
index 71b14eb..34b67bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedMultiNodeFullApiSelfTest.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteAtomicLong;
import org.apache.ignite.IgniteCache;
@@ -37,10 +38,12 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.testframework.GridTestUtils;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
@@ -194,7 +197,7 @@ public class GridCachePartitionedMultiNodeFullApiSelfTest extends GridCacheParti
for (int i = 0; i < gridCount(); i++)
grid(i).events().localListen(
- new SwapUnswapLocalListener(), EVT_CACHE_OBJECT_SWAPPED, EVT_CACHE_OBJECT_UNSWAPPED);
+ new SwapUnswapLocalListener(), EVT_CACHE_OBJECT_SWAPPED, EVT_CACHE_OBJECT_UNSWAPPED);
jcache().put("key", 1);
@@ -202,13 +205,19 @@ public class GridCachePartitionedMultiNodeFullApiSelfTest extends GridCacheParti
if (grid(i).affinity(null).isBackup(grid(i).localNode(), "key")) {
jcache(i).localEvict(Collections.singleton("key"));
- assert jcache(i).localPeek("key", ONHEAP) == null;
+ assertNull(jcache(i).localPeek("key", ONHEAP));
- assert jcache(i).get("key") == 1;
+ assertEquals((Integer)1, jcache(i).get("key"));
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return swapEvts.get() == 1 && unswapEvts.get() == 1;
+ }
+ }, 5000);
- assert swapEvts.get() == 1 : "Swap events: " + swapEvts.get();
+ assertEquals(1, swapEvts.get());
- assert unswapEvts.get() == 1 : "Unswap events: " + unswapEvts.get();
+ assertEquals(1, unswapEvts.get());
break;
}
@@ -464,11 +473,25 @@ public class GridCachePartitionedMultiNodeFullApiSelfTest extends GridCacheParti
switch (evt.type()) {
case EVT_CACHE_OBJECT_SWAPPED:
- ignite.atomicLong("swapEvts", 0, false).incrementAndGet();
+ // Run from another thread to avoid deadlock with striped pool.
+ GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ignite.atomicLong("swapEvts", 0, false).incrementAndGet();
+
+ return null;
+ }
+ });
break;
case EVT_CACHE_OBJECT_UNSWAPPED:
- ignite.atomicLong("unswapEvts", 0, false).incrementAndGet();
+ // Run from another thread to avoid deadlock with striped pool.
+ GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ignite.atomicLong("unswapEvts", 0, false).incrementAndGet();
+
+ return null;
+ }
+ });
break;
}
[50/50] [abbrv] ignite git commit: Merge ignite-1.8.2.
Posted by sb...@apache.org.
Merge ignite-1.8.2.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1b2afbb8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1b2afbb8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1b2afbb8
Branch: refs/heads/master
Commit: 1b2afbb8d4032d23ee9a78987aec90a42e886700
Parents: 41dddb8 8372e69
Author: sboikov <sb...@gridgain.com>
Authored: Thu Dec 22 16:51:00 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Dec 22 16:51:00 2016 +0300
----------------------------------------------------------------------
.../store/cassandra/CassandraCacheStore.java | 6 +
.../store/cassandra/datasource/DataSource.java | 9 +
.../java/org/apache/ignite/IgniteServices.java | 16 +
.../ignite/cache/store/CacheStoreAdapter.java | 6 +
.../cache/store/jdbc/CacheJdbcPojoStore.java | 19 +-
.../store/jdbc/JdbcTypesDefaultTransformer.java | 112 ++--
.../org/apache/ignite/internal/GridTopic.java | 8 +-
.../ignite/internal/IgniteServicesImpl.java | 9 +-
.../internal/binary/BinaryClassDescriptor.java | 12 +-
.../ignite/internal/binary/BinaryUtils.java | 10 +-
.../binary/builder/BinaryObjectBuilderImpl.java | 11 +-
.../affinity/GridAffinityProcessor.java | 2 +-
.../cache/CacheStoreBalancingWrapper.java | 6 +
.../processors/cache/GridCacheAdapter.java | 8 +-
.../cache/GridCacheLoaderWriterStore.java | 6 +
.../GridCachePartitionExchangeManager.java | 23 +-
.../CacheDataStructuresManager.java | 6 +-
.../cache/query/GridCacheQueryManager.java | 4 +-
.../closure/GridClosureProcessor.java | 31 +-
.../internal/processors/job/GridJobWorker.java | 76 ++-
.../processors/odbc/OdbcRequestHandler.java | 14 +-
.../platform/PlatformContextImpl.java | 2 +-
.../dotnet/PlatformDotNetCacheStore.java | 11 +
.../platform/services/PlatformServices.java | 2 +-
.../platform/utils/PlatformUtils.java | 28 +
.../service/GridServiceProcessor.java | 11 +-
.../processors/service/GridServiceProxy.java | 18 +-
.../processors/task/GridTaskWorker.java | 7 +
.../communication/tcp/TcpCommunicationSpi.java | 4 +-
.../jdbc/JdbcTypesDefaultTransformerTest.java | 283 +++++++++
.../IgniteComputeTopologyExceptionTest.java | 5 +-
.../binary/BinaryMarshallerSelfTest.java | 66 ++
...heapCacheMetricsForClusterGroupSelfTest.java | 141 +++++
.../CacheOffHeapAndSwapMetricsSelfTest.java | 621 -------------------
...LocalCacheOffHeapAndSwapMetricsSelfTest.java | 621 +++++++++++++++++++
.../closure/GridClosureSerializationTest.java | 177 ++++++
...gniteServiceProxyTimeoutInitializedTest.java | 284 +++++++++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
.../IgniteCacheMetricsSelfTestSuite.java | 6 +-
.../ignite/testsuites/IgniteCacheTestSuite.java | 2 +
.../testsuites/IgniteKernalSelfTestSuite.java | 2 +
.../processors/query/h2/IgniteH2Indexing.java | 75 +--
...niteCachePartitionedFieldsQuerySelfTest.java | 25 +
43 files changed, 2008 insertions(+), 779 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b2afbb8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b2afbb8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b2afbb8/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
index ea3ae8b,c6b41d2..2e0fd10
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
@@@ -198,13 -198,19 +198,19 @@@ public class OdbcRequestHandler
qry.setDistributedJoins(distributedJoins);
qry.setEnforceJoinOrder(enforceJoinOrder);
- IgniteCache<Object, Object> cache = ctx.grid().cache(req.cacheName());
+ IgniteCache<Object, Object> cache0 = ctx.grid().cache(req.cacheName());
+
+ if (cache0 == null)
+ return new OdbcResponse(OdbcResponse.STATUS_FAILED,
- "Cache doesn't exist (did you configure it?): " + req.cacheName());
++ "Cache doesn't exist (did you configure it?): " + req.cacheName());
+
+ IgniteCache<Object, Object> cache = cache0.withKeepBinary();
if (cache == null)
- return new OdbcResponse(OdbcResponse.STATUS_FAILED, "Cache doesn't exist (did you configure it?): " +
- req.cacheName());
+ return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+ "Can not get cache with keep binary: " + req.cacheName());
- QueryCursor qryCur = cache.withKeepBinary().query(qry);
+ QueryCursor qryCur = cache.query(qry);
qryCursors.put(qryId, new IgniteBiTuple<QueryCursor, Iterator>(qryCur, null));
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b2afbb8/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b2afbb8/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index 22a7fa2,c266986..37727f5
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@@ -341,10 -340,10 +341,10 @@@ public class PlatformServices extends P
Object proxy = PlatformService.class.isAssignableFrom(d.serviceClass())
? services.serviceProxy(name, PlatformService.class, sticky)
- : new GridServiceProxy<>(services.clusterGroup(), name, Service.class, sticky,
+ : new GridServiceProxy<>(services.clusterGroup(), name, Service.class, sticky, 0,
platformCtx.kernalContext());
- return new ServiceProxyHolder(proxy, d.serviceClass());
+ return new ServiceProxyHolder(proxy, d.serviceClass(), platformContext());
}
}
return super.processInStreamOutObject(type, reader);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1b2afbb8/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
----------------------------------------------------------------------
[42/50] [abbrv] ignite git commit: ignite-4371 Neat TX finish request
processing may fall into sync wait of dht finish response - enabled striped
pool usage for GridDistributedTxFinishResponse - convert exceptions from
CacheStoreSessionListener.onSession
Posted by sb...@apache.org.
ignite-4371 Neat TX finish request processing may fall into sync wait of dht finish response
- enabled striped pool usage for GridDistributedTxFinishResponse
- convert exceptions from CacheStoreSessionListener.onSessionStart into IgniteCheckedException
- should not use IgniteTxLocalAdapter.commitErr for rollback exceptions (otherwise cyclic reference in exception->cause is possible, this leads to infinite loop in 'X.hasCause')
(cherry picked from commit cf7b6a0)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/147277d8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/147277d8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/147277d8
Branch: refs/heads/master
Commit: 147277d89fcdbd134425b74fc0b63596572a4af1
Parents: c1fde05
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 19 14:49:25 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 19 15:17:38 2016 +0300
----------------------------------------------------------------------
.../GridDistributedTxFinishResponse.java | 6 ----
.../cache/store/CacheStoreManager.java | 4 ++-
.../store/GridCacheStoreManagerAdapter.java | 34 ++++++++++++--------
.../cache/store/GridCacheWriteBehindStore.java | 6 ++--
.../transactions/IgniteTxLocalAdapter.java | 27 ++++++----------
5 files changed, 36 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/147277d8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
index c5cf332..109d665 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
@@ -86,12 +86,6 @@ public class GridDistributedTxFinishResponse extends GridCacheMessage {
}
/** {@inheritDoc} */
- @Override public int partition() {
- // TODO https://issues.apache.org/jira/browse/IGNITE-4371
- return Integer.MIN_VALUE;
- }
-
- /** {@inheritDoc} */
@Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
writer.setBuffer(buf);
http://git-wip-us.apache.org/repos/asf/ignite/blob/147277d8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
index 67c9334..8d6b63d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/CacheStoreManager.java
@@ -171,8 +171,10 @@ public interface CacheStoreManager<K, V> extends GridCacheManager<K, V> {
/**
* End session initiated by write-behind store.
+ *
+ * @throws IgniteCheckedException If failed.
*/
- public void writeBehindSessionInit();
+ public void writeBehindSessionInit() throws IgniteCheckedException;
/**
* End session initiated by write-behind store.
http://git-wip-us.apache.org/repos/asf/ignite/blob/147277d8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 024375e..8b93afa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -288,7 +288,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override @Nullable public Object load(@Nullable IgniteInternalTx tx, KeyCacheObject key)
+ @Override @Nullable public final Object load(@Nullable IgniteInternalTx tx, KeyCacheObject key)
throws IgniteCheckedException {
return loadFromStore(tx, key, true);
}
@@ -377,7 +377,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public void localStoreLoadAll(@Nullable IgniteInternalTx tx, Collection keys, GridInClosure3 vis)
+ @Override public final void localStoreLoadAll(@Nullable IgniteInternalTx tx, Collection keys, GridInClosure3 vis)
throws IgniteCheckedException {
assert store != null;
assert locStore;
@@ -386,7 +386,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public boolean loadAll(@Nullable IgniteInternalTx tx, Collection keys, IgniteBiInClosure vis)
+ @Override public final boolean loadAll(@Nullable IgniteInternalTx tx, Collection keys, IgniteBiInClosure vis)
throws IgniteCheckedException {
if (store != null) {
loadAllFromStore(tx, keys, vis, null);
@@ -499,7 +499,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public boolean loadCache(final GridInClosure3 vis, Object[] args) throws IgniteCheckedException {
+ @Override public final boolean loadCache(final GridInClosure3 vis, Object[] args) throws IgniteCheckedException {
if (store != null) {
if (log.isDebugEnabled())
log.debug("Loading all values from store.");
@@ -554,7 +554,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public boolean put(@Nullable IgniteInternalTx tx, Object key, Object val, GridCacheVersion ver)
+ @Override public final boolean put(@Nullable IgniteInternalTx tx, Object key, Object val, GridCacheVersion ver)
throws IgniteCheckedException {
if (store != null) {
// Never persist internal keys.
@@ -599,7 +599,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public boolean putAll(@Nullable IgniteInternalTx tx, Map map) throws IgniteCheckedException {
+ @Override public final boolean putAll(@Nullable IgniteInternalTx tx, Map map) throws IgniteCheckedException {
if (F.isEmpty(map))
return true;
@@ -658,7 +658,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public boolean remove(@Nullable IgniteInternalTx tx, Object key) throws IgniteCheckedException {
+ @Override public final boolean remove(@Nullable IgniteInternalTx tx, Object key) throws IgniteCheckedException {
if (store != null) {
// Never remove internal key from store as it is never persisted.
if (key instanceof GridCacheInternal)
@@ -701,7 +701,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public boolean removeAll(@Nullable IgniteInternalTx tx, Collection keys) throws IgniteCheckedException {
+ @Override public final boolean removeAll(@Nullable IgniteInternalTx tx, Collection keys) throws IgniteCheckedException {
if (F.isEmpty(keys))
return true;
@@ -763,7 +763,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
+ @Override public final void sessionEnd(IgniteInternalTx tx, boolean commit, boolean last) throws IgniteCheckedException {
assert store != null;
sessionInit0(tx);
@@ -807,7 +807,7 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
}
/** {@inheritDoc} */
- @Override public void writeBehindSessionInit() {
+ @Override public void writeBehindSessionInit() throws IgniteCheckedException {
sessionInit0(null);
}
@@ -818,8 +818,9 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
/**
* @param tx Current transaction.
+ * @throws IgniteCheckedException If failed.
*/
- private void sessionInit0(@Nullable IgniteInternalTx tx) {
+ private void sessionInit0(@Nullable IgniteInternalTx tx) throws IgniteCheckedException {
assert sesHolder != null;
SessionData ses;
@@ -841,9 +842,14 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
sesHolder.set(ses);
- if (sesLsnrs != null && !ses.started(this)) {
- for (CacheStoreSessionListener lsnr : sesLsnrs)
- lsnr.onSessionStart(locSes);
+ try {
+ if (sesLsnrs != null && !ses.started(this)) {
+ for (CacheStoreSessionListener lsnr : sesLsnrs)
+ lsnr.onSessionStart(locSes);
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteCheckedException("Failed to start store session: " + e, e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/147277d8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 858d9a7..f5c0c12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -673,10 +673,10 @@ public class GridCacheWriteBehindStore<K, V> implements CacheStore<K, V>, Lifecy
Map<K, Entry<? extends K, ? extends V>> vals,
boolean initSes) {
- if (initSes && storeMgr != null)
- storeMgr.writeBehindSessionInit();
-
try {
+ if (initSes && storeMgr != null)
+ storeMgr.writeBehindSessionInit();
+
boolean threwEx = true;
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/147277d8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 8d0a2b1..9f794b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1063,7 +1063,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
setRollbackOnly();
throw new IgniteCheckedException("Invalid transaction state for rollback [state=" + state +
- ", tx=" + this + ']', commitErr);
+ ", tx=" + this + ']');
}
if (near()) {
@@ -1074,28 +1074,21 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
- try {
- cctx.tm().rollbackTx(this);
+ cctx.tm().rollbackTx(this);
- if (!internal()) {
- Collection<CacheStoreManager> stores = txState.stores(cctx);
+ if (!internal()) {
+ Collection<CacheStoreManager> stores = txState.stores(cctx);
- if (stores != null && !stores.isEmpty()) {
- assert isWriteToStoreFromDhtValid(stores) :
- "isWriteToStoreFromDht can't be different within one transaction";
+ if (stores != null && !stores.isEmpty()) {
+ assert isWriteToStoreFromDhtValid(stores) :
+ "isWriteToStoreFromDht can't be different within one transaction";
- boolean isWriteToStoreFromDht = F.first(stores).isWriteToStoreFromDht();
+ boolean isWriteToStoreFromDht = F.first(stores).isWriteToStoreFromDht();
- if (stores != null && !stores.isEmpty() && (near() || isWriteToStoreFromDht))
- sessionEnd(stores, false);
- }
+ if (!stores.isEmpty() && (near() || isWriteToStoreFromDht))
+ sessionEnd(stores, false);
}
}
- catch (Error | IgniteCheckedException | RuntimeException e) {
- U.addLastCause(e, commitErr, log);
-
- throw e;
- }
}
}
[33/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1fe437c..0c90414 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -46,6 +46,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLException;
import org.apache.ignite.Ignite;
@@ -53,6 +54,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.AddressResolver;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -103,6 +105,7 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.extensions.communication.Message;
@@ -179,6 +182,7 @@ import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META
* <li>Node local IP address (see {@link #setLocalAddress(String)})</li>
* <li>Node local port number (see {@link #setLocalPort(int)})</li>
* <li>Local port range (see {@link #setLocalPortRange(int)}</li>
+ * <li>Connections per node (see {@link #setConnectionsPerNode(int)})</li>
* <li>Connection buffer flush frequency (see {@link #setConnectionBufferFlushFrequency(long)})</li>
* <li>Connection buffer size (see {@link #setConnectionBufferSize(int)})</li>
* <li>Idle connection timeout (see {@link #setIdleConnectionTimeout(long)})</li>
@@ -238,6 +242,9 @@ import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META
@IgniteSpiConsistencyChecked(optional = false)
public class TcpCommunicationSpi extends IgniteSpiAdapter
implements CommunicationSpi<Message>, TcpCommunicationSpiMBean {
+ /** */
+ private static final IgniteProductVersion MULTIPLE_CONN_SINCE_VER = IgniteProductVersion.fromString("1.8.2");
+
/** IPC error message. */
public static final String OUT_OF_RESOURCES_TCP_MSG = "Failed to allocate shared memory segment " +
"(switching to TCP, may be slower).";
@@ -257,11 +264,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/** Node attribute that is mapped to node's external addresses (value is <tt>comm.tcp.ext-addrs</tt>). */
public static final String ATTR_EXT_ADDRS = "comm.tcp.ext-addrs";
+ /** */
+ public static final String ATTR_PAIRED_CONN = "comm.tcp.pairedConnection";
+
/** Default port which node sets listener to (value is <tt>47100</tt>). */
public static final int DFLT_PORT = 47100;
/** Default port which node sets listener for shared memory connections (value is <tt>48100</tt>). */
- public static final int DFLT_SHMEM_PORT = 48100;
+ public static final int DFLT_SHMEM_PORT = -1;
/** Default idle connection timeout (value is <tt>30000</tt>ms). */
public static final long DFLT_IDLE_CONN_TIMEOUT = 30000;
@@ -283,12 +293,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/**
* Default count of selectors for TCP server equals to
- * {@code "Math.min(4, Runtime.getRuntime().availableProcessors())"}.
+ * {@code "Math.min(8, Runtime.getRuntime().availableProcessors())"}.
*/
- public static final int DFLT_SELECTORS_CNT = Math.min(4, Runtime.getRuntime().availableProcessors());
+ public static final int DFLT_SELECTORS_CNT = Math.max(4, Runtime.getRuntime().availableProcessors() / 2);
- /** Node ID meta for session. */
- private static final int NODE_ID_META = GridNioSessionMetaKey.nextUniqueKey();
+ /** Connection index meta for session. */
+ private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
/** Message tracker meta for session. */
private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
@@ -303,11 +313,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
public static final boolean DFLT_TCP_NODELAY = true;
/** Default received messages threshold for sending ack. */
- public static final int DFLT_ACK_SND_THRESHOLD = 16;
+ public static final int DFLT_ACK_SND_THRESHOLD = 32;
/** Default socket write timeout. */
public static final long DFLT_SOCK_WRITE_TIMEOUT = 2000;
+ /** Default connections per node. */
+ public static final int DFLT_CONN_PER_NODE = 1;
+
/** No-op runnable. */
private static final IgniteRunnable NOOP = new IgniteRunnable() {
@Override public void run() {
@@ -327,11 +340,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/** */
private ConnectGateway connectGate;
+ /** */
+ private ConnectionPolicy connPlc;
+
/** Server listener. */
private final GridNioServerListener<Message> srvLsnr =
new GridNioServerListenerAdapter<Message>() {
@Override public void onSessionWriteTimeout(GridNioSession ses) {
- LT.warn(log, "Communication SPI Session write timed out (consider increasing " +
+ LT.warn(log,"Communication SPI session write timed out (consider increasing " +
"'socketWriteTimeout' " + "configuration property) [remoteAddr=" + ses.remoteAddress() +
", writeTimeout=" + sockWriteTimeout + ']');
@@ -347,46 +363,53 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
if (log.isDebugEnabled())
log.debug("Sending local node ID to newly accepted session: " + ses);
- ses.send(nodeIdMessage());
+ try {
+ ses.sendNoFuture(nodeIdMessage());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send message: " + e, e);
+ }
}
}
@Override public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
- UUID id = ses.meta(NODE_ID_META);
+ ConnectionKey connId = ses.meta(CONN_IDX_META);
- if (id != null) {
- GridCommunicationClient client = clients.get(id);
+ if (connId != null) {
+ UUID id = connId.nodeId();
- if (client instanceof GridTcpNioCommunicationClient &&
- ((GridTcpNioCommunicationClient) client).session() == ses) {
- client.close();
+ GridCommunicationClient[] nodeClients = clients.get(id);
- clients.remove(id, client);
+ if (nodeClients != null) {
+ for (GridCommunicationClient client : nodeClients) {
+ if (client instanceof GridTcpNioCommunicationClient &&
+ ((GridTcpNioCommunicationClient)client).session() == ses) {
+ client.close();
+
+ removeNodeClient(id, client);
+ }
+ }
}
if (!stopping) {
- boolean reconnect = false;
-
- GridNioRecoveryDescriptor recoveryData = ses.recoveryDescriptor();
-
- if (recoveryData != null) {
- if (recoveryData.nodeAlive(getSpiContext().node(id))) {
- if (!recoveryData.messagesFutures().isEmpty()) {
- reconnect = true;
+ GridNioRecoveryDescriptor outDesc = ses.outRecoveryDescriptor();
+ if (outDesc != null) {
+ if (outDesc.nodeAlive(getSpiContext().node(id))) {
+ if (!outDesc.messagesRequests().isEmpty()) {
if (log.isDebugEnabled())
log.debug("Session was closed but there are unacknowledged messages, " +
- "will try to reconnect [rmtNode=" + recoveryData.node().id() + ']');
+ "will try to reconnect [rmtNode=" + outDesc.node().id() + ']');
+
+ DisconnectedSessionInfo disconnectData =
+ new DisconnectedSessionInfo(outDesc, connId.connectionIndex());
+
+ commWorker.addProcessDisconnectRequest(disconnectData);
}
}
else
- recoveryData.onNodeLeft();
+ outDesc.onNodeLeft();
}
-
- DisconnectedSessionInfo disconnectData = new DisconnectedSessionInfo(recoveryData,
- reconnect);
-
- commWorker.addProcessDisconnectRequest(disconnectData);
}
CommunicationListener<Message> lsnr0 = lsnr;
@@ -403,21 +426,24 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
private void onFirstMessage(GridNioSession ses, Message msg) {
UUID sndId;
- if (msg instanceof NodeIdMessage)
- sndId = U.bytesToUuid(((NodeIdMessage)msg).nodeIdBytes, 0);
+ ConnectionKey connKey;
+
+ if (msg instanceof NodeIdMessage) {
+ sndId = U.bytesToUuid(((NodeIdMessage) msg).nodeIdBytes, 0);
+ connKey = new ConnectionKey(sndId, 0, -1);
+ }
else {
assert msg instanceof HandshakeMessage : msg;
+ HandshakeMessage msg0 = (HandshakeMessage)msg;
+
sndId = ((HandshakeMessage)msg).nodeId();
+ connKey = new ConnectionKey(sndId, msg0.connectionIndex(), msg0.connectCount());
}
if (log.isDebugEnabled())
log.debug("Remote node ID received: " + sndId);
- final UUID old = ses.addMeta(NODE_ID_META, sndId);
-
- assert old == null;
-
final ClusterNode rmtNode = getSpiContext().node(sndId);
if (rmtNode == null) {
@@ -429,57 +455,65 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
return;
}
+ final ConnectionKey old = ses.addMeta(CONN_IDX_META, connKey);
+
+ assert old == null;
+
ClusterNode locNode = getSpiContext().localNode();
if (ses.remoteAddress() == null)
return;
- GridCommunicationClient oldClient = clients.get(sndId);
+ assert msg instanceof HandshakeMessage : msg;
- boolean hasShmemClient = false;
+ HandshakeMessage msg0 = (HandshakeMessage)msg;
- if (oldClient != null) {
- if (oldClient instanceof GridTcpNioCommunicationClient) {
- if (log.isDebugEnabled())
- log.debug("Received incoming connection when already connected " +
- "to this node, rejecting [locNode=" + locNode.id() +
- ", rmtNode=" + sndId + ']');
+ if (usePairedConnections(rmtNode)) {
+ final GridNioRecoveryDescriptor recoveryDesc = inRecoveryDescriptor(rmtNode, connKey);
- ses.send(new RecoveryLastReceivedMessage(-1));
+ ConnectClosureNew c = new ConnectClosureNew(ses, recoveryDesc, rmtNode);
- return;
- }
+ boolean reserve = recoveryDesc.tryReserve(msg0.connectCount(), c);
+
+ if (reserve)
+ connectedNew(recoveryDesc, ses, true);
else {
- assert oldClient instanceof GridShmemCommunicationClient;
+ if (c.failed) {
+ ses.send(new RecoveryLastReceivedMessage(-1));
+
+ for (GridNioSession ses0 : nioSrvr.sessions()) {
+ ConnectionKey key0 = ses0.meta(CONN_IDX_META);
- hasShmemClient = true;
+ if (ses0.accepted() && key0 != null &&
+ key0.nodeId().equals(connKey.nodeId()) &&
+ key0.connectionIndex() == connKey.connectionIndex() &&
+ key0.connectCount() < connKey.connectCount())
+ ses0.close();
+ }
+ }
}
}
+ else {
+ assert connKey.connectionIndex() >= 0 : connKey;
- GridFutureAdapter<GridCommunicationClient> fut = new GridFutureAdapter<>();
-
- GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(sndId, fut);
-
- assert msg instanceof HandshakeMessage : msg;
-
- HandshakeMessage msg0 = (HandshakeMessage)msg;
+ GridCommunicationClient[] curClients = clients.get(sndId);
- final GridNioRecoveryDescriptor recoveryDesc = recoveryDescriptor(rmtNode);
+ GridCommunicationClient oldClient =
+ curClients != null && connKey.connectionIndex() < curClients.length ?
+ curClients[connKey.connectionIndex()] :
+ null;
- if (oldFut == null) {
- oldClient = clients.get(sndId);
+ boolean hasShmemClient = false;
if (oldClient != null) {
if (oldClient instanceof GridTcpNioCommunicationClient) {
if (log.isDebugEnabled())
log.debug("Received incoming connection when already connected " +
- "to this node, rejecting [locNode=" + locNode.id() +
- ", rmtNode=" + sndId + ']');
+ "to this node, rejecting [locNode=" + locNode.id() +
+ ", rmtNode=" + sndId + ']');
ses.send(new RecoveryLastReceivedMessage(-1));
- fut.onDone(oldClient);
-
return;
}
else {
@@ -489,51 +523,86 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
}
}
- boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
- new ConnectClosure(ses, recoveryDesc, rmtNode, msg0, !hasShmemClient, fut));
+ GridFutureAdapter<GridCommunicationClient> fut = new GridFutureAdapter<>();
- if (log.isDebugEnabled())
- log.debug("Received incoming connection from remote node " +
+ GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(connKey, fut);
+
+ final GridNioRecoveryDescriptor recoveryDesc = inRecoveryDescriptor(rmtNode, connKey);
+
+ if (oldFut == null) {
+ curClients = clients.get(sndId);
+
+ oldClient = curClients != null && connKey.connectionIndex() < curClients.length ?
+ curClients[connKey.connectionIndex()] : null;
+
+ if (oldClient != null) {
+ if (oldClient instanceof GridTcpNioCommunicationClient) {
+ assert oldClient.connectionIndex() == connKey.connectionIndex() : oldClient;
+
+ if (log.isDebugEnabled())
+ log.debug("Received incoming connection when already connected " +
+ "to this node, rejecting [locNode=" + locNode.id() +
+ ", rmtNode=" + sndId + ']');
+
+ ses.send(new RecoveryLastReceivedMessage(-1));
+
+ fut.onDone(oldClient);
+
+ return;
+ }
+ else {
+ assert oldClient instanceof GridShmemCommunicationClient;
+
+ hasShmemClient = true;
+ }
+ }
+
+ boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
+ new ConnectClosure(ses, recoveryDesc, rmtNode, connKey, msg0, !hasShmemClient, fut));
+
+ if (log.isDebugEnabled())
+ log.debug("Received incoming connection from remote node " +
"[rmtNode=" + rmtNode.id() + ", reserved=" + reserved + ']');
- if (reserved) {
- try {
- GridTcpNioCommunicationClient client =
+ if (reserved) {
+ try {
+ GridTcpNioCommunicationClient client =
connected(recoveryDesc, ses, rmtNode, msg0.received(), true, !hasShmemClient);
- fut.onDone(client);
- }
- finally {
- clientFuts.remove(rmtNode.id(), fut);
+ fut.onDone(client);
+ }
+ finally {
+ clientFuts.remove(connKey, fut);
+ }
}
}
- }
- else {
- if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
- if (log.isDebugEnabled()) {
- log.debug("Received incoming connection from remote node while " +
+ else {
+ if (oldFut instanceof ConnectFuture && locNode.order() < rmtNode.order()) {
+ if (log.isDebugEnabled()) {
+ log.debug("Received incoming connection from remote node while " +
"connecting to this node, rejecting [locNode=" + locNode.id() +
", locNodeOrder=" + locNode.order() + ", rmtNode=" + rmtNode.id() +
", rmtNodeOrder=" + rmtNode.order() + ']');
- }
+ }
- ses.send(new RecoveryLastReceivedMessage(-1));
- }
- else {
- // The code below causes a race condition between shmem and TCP (see IGNITE-1294)
- boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
- new ConnectClosure(ses, recoveryDesc, rmtNode, msg0, !hasShmemClient, fut));
+ ses.send(new RecoveryLastReceivedMessage(-1));
+ }
+ else {
+ // The code below causes a race condition between shmem and TCP (see IGNITE-1294)
+ boolean reserved = recoveryDesc.tryReserve(msg0.connectCount(),
+ new ConnectClosure(ses, recoveryDesc, rmtNode, connKey, msg0, !hasShmemClient, fut));
- if (reserved)
- connected(recoveryDesc, ses, rmtNode, msg0.received(), true, !hasShmemClient);
+ if (reserved)
+ connected(recoveryDesc, ses, rmtNode, msg0.received(), true, !hasShmemClient);
+ }
}
}
}
@Override public void onMessage(GridNioSession ses, Message msg) {
- UUID sndId = ses.meta(NODE_ID_META);
+ ConnectionKey connKey = ses.meta(CONN_IDX_META);
- if (sndId == null) {
+ if (connKey == null) {
assert ses.accepted() : ses;
if (!connectGate.tryEnter()) {
@@ -555,29 +624,37 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
else {
rcvdMsgsCnt.increment();
- GridNioRecoveryDescriptor recovery = ses.recoveryDescriptor();
+ if (msg instanceof RecoveryLastReceivedMessage) {
+ GridNioRecoveryDescriptor recovery = ses.outRecoveryDescriptor();
- if (recovery != null) {
- if (msg instanceof RecoveryLastReceivedMessage) {
+ if (recovery != null) {
RecoveryLastReceivedMessage msg0 = (RecoveryLastReceivedMessage)msg;
- if (log.isDebugEnabled())
- log.debug("Received recovery acknowledgement [rmtNode=" + sndId +
+ if (log.isDebugEnabled()) {
+ log.debug("Received recovery acknowledgement [rmtNode=" + connKey.nodeId() +
+ ", connIdx=" + connKey.connectionIndex() +
", rcvCnt=" + msg0.received() + ']');
+ }
recovery.ackReceived(msg0.received());
return;
}
- else {
+ }
+ else {
+ GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
+
+ if (recovery != null) {
long rcvCnt = recovery.onReceived();
if (rcvCnt % ackSndThreshold == 0) {
- if (log.isDebugEnabled())
- log.debug("Send recovery acknowledgement [rmtNode=" + sndId +
+ if (log.isDebugEnabled()) {
+ log.debug("Send recovery acknowledgement [rmtNode=" + connKey.nodeId() +
+ ", connIdx=" + connKey.connectionIndex() +
", rcvCnt=" + rcvCnt + ']');
+ }
- nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(rcvCnt));
+ ses.systemMessage(new RecoveryLastReceivedMessage(rcvCnt));
recovery.lastAcknowledged(rcvCnt);
}
@@ -603,7 +680,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
else
c = NOOP;
- notifyListener(sndId, msg, c);
+ notifyListener(connKey.nodeId(), msg, c);
}
}
@@ -611,7 +688,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
* @param recovery Recovery descriptor.
* @param ses Session.
* @param node Node.
- * @param rcvCnt Number of received messages..
+ * @param rcvCnt Number of received messages.
* @param sndRes If {@code true} sends response for recovery handshake.
* @param createClient If {@code true} creates NIO communication client.
* @return Client.
@@ -623,32 +700,128 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
long rcvCnt,
boolean sndRes,
boolean createClient) {
+ ConnectionKey connKey = ses.meta(CONN_IDX_META);
+
+ assert connKey != null && connKey.connectionIndex() >= 0 : connKey;
+ assert !usePairedConnections(node);
+
recovery.onHandshake(rcvCnt);
- ses.recoveryDescriptor(recovery);
+ ses.inRecoveryDescriptor(recovery);
+ ses.outRecoveryDescriptor(recovery);
nioSrvr.resend(ses);
- if (sndRes)
- nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recovery.received()));
+ try {
+ if (sndRes)
+ nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recovery.received()));
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send message: " + e, e);
+ }
- recovery.connected();
+ recovery.onConnected();
GridTcpNioCommunicationClient client = null;
if (createClient) {
- client = new GridTcpNioCommunicationClient(ses, log);
-
- GridCommunicationClient oldClient = clients.putIfAbsent(node.id(), client);
+ client = new GridTcpNioCommunicationClient(connKey.connectionIndex(), ses, log);
- assert oldClient == null : "Client already created [node=" + node + ", client=" + client +
- ", oldClient=" + oldClient + ", recoveryDesc=" + recovery + ']';
+ addNodeClient(node, connKey.connectionIndex(), client);
}
return client;
}
/**
+ * @param recovery Recovery descriptor.
+ * @param ses Session.
+ * @param sndRes If {@code true} sends response for recovery handshake.
+ */
+ private void connectedNew(
+ GridNioRecoveryDescriptor recovery,
+ GridNioSession ses,
+ boolean sndRes) {
+ try {
+ ses.inRecoveryDescriptor(recovery);
+
+ if (sndRes)
+ nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recovery.received()));
+
+ recovery.onConnected();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send message: " + e, e);
+ }
+ }
+
+ /**
+ *
+ */
+ class ConnectClosureNew implements IgniteInClosure<Boolean> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final GridNioSession ses;
+
+ /** */
+ private final GridNioRecoveryDescriptor recoveryDesc;
+
+ /** */
+ private final ClusterNode rmtNode;
+
+ /** */
+ private boolean failed;
+
+ /**
+ * @param ses Incoming session.
+ * @param recoveryDesc Recovery descriptor.
+ * @param rmtNode Remote node.
+ */
+ ConnectClosureNew(GridNioSession ses,
+ GridNioRecoveryDescriptor recoveryDesc,
+ ClusterNode rmtNode) {
+ this.ses = ses;
+ this.recoveryDesc = recoveryDesc;
+ this.rmtNode = rmtNode;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void apply(Boolean success) {
+ try {
+ failed = !success;
+
+ if (success) {
+ IgniteInClosure<IgniteInternalFuture<?>> lsnr = new IgniteInClosure<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> msgFut) {
+ try {
+ msgFut.get();
+
+ connectedNew(recoveryDesc, ses, false);
+ }
+ catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send recovery handshake " +
+ "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
+
+ recoveryDesc.release();
+ }
+ }
+ };
+
+ nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recoveryDesc.received()), lsnr);
+ }
+ else
+ nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(-1));
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send message: " + e, e);
+ }
+ }
+ }
+
+ /**
*
*/
@SuppressWarnings("PackageVisibleInnerClass")
@@ -674,10 +847,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/** */
private final boolean createClient;
+ /** */
+ private final ConnectionKey connKey;
+
/**
* @param ses Incoming session.
* @param recoveryDesc Recovery descriptor.
* @param rmtNode Remote node.
+ * @param connKey Connection key.
* @param msg Handshake message.
* @param createClient If {@code true} creates NIO communication client..
* @param fut Connect future.
@@ -685,12 +862,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
ConnectClosure(GridNioSession ses,
GridNioRecoveryDescriptor recoveryDesc,
ClusterNode rmtNode,
+ ConnectionKey connKey,
HandshakeMessage msg,
boolean createClient,
GridFutureAdapter<GridCommunicationClient> fut) {
this.ses = ses;
this.recoveryDesc = recoveryDesc;
this.rmtNode = rmtNode;
+ this.connKey = connKey;
this.msg = msg;
this.createClient = createClient;
this.fut = fut;
@@ -699,39 +878,44 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/** {@inheritDoc} */
@Override public void apply(Boolean success) {
if (success) {
- IgniteInClosure<IgniteInternalFuture<?>> lsnr = new IgniteInClosure<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> msgFut) {
- try {
- msgFut.get();
+ try {
+ IgniteInClosure<IgniteInternalFuture<?>> lsnr = new IgniteInClosure<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> msgFut) {
+ try {
+ msgFut.get();
- GridTcpNioCommunicationClient client =
- connected(recoveryDesc, ses, rmtNode, msg.received(), false, createClient);
+ GridTcpNioCommunicationClient client =
+ connected(recoveryDesc, ses, rmtNode, msg.received(), false, createClient);
- fut.onDone(client);
- }
- catch (IgniteCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to send recovery handshake " +
- "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
+ fut.onDone(client);
+ }
+ catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to send recovery handshake " +
+ "[rmtNode=" + rmtNode.id() + ", err=" + e + ']');
- recoveryDesc.release();
+ recoveryDesc.release();
- fut.onDone();
- }
- finally {
- clientFuts.remove(rmtNode.id(), fut);
+ fut.onDone();
+ }
+ finally {
+ clientFuts.remove(connKey, fut);
+ }
}
- }
- };
+ };
- nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recoveryDesc.received()), lsnr);
+ nioSrvr.sendSystem(ses, new RecoveryLastReceivedMessage(recoveryDesc.received()), lsnr);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send message: " + e, e);
+ }
}
else {
try {
fut.onDone();
}
finally {
- clientFuts.remove(rmtNode.id(), fut);
+ clientFuts.remove(connKey, fut);
}
}
}
@@ -794,6 +978,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/** Shared memory server. */
private IpcSharedMemoryServerEndpoint shmemSrv;
+ /** */
+ private boolean usePairedConnections = true;
+
+ /** */
+ private int connectionsPerNode = DFLT_CONN_PER_NODE;
+
/** {@code TCP_NODELAY} option value for created sockets. */
private boolean tcpNoDelay = DFLT_TCP_NODELAY;
@@ -816,7 +1006,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
private final Collection<ShmemWorker> shmemWorkers = new ConcurrentLinkedDeque8<>();
/** Clients. */
- private final ConcurrentMap<UUID, GridCommunicationClient> clients = GridConcurrentFactory.newMap();
+ private final ConcurrentMap<UUID, GridCommunicationClient[]> clients = GridConcurrentFactory.newMap();
/** SPI listener. */
private volatile CommunicationListener<Message> lsnr;
@@ -830,6 +1020,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/** Count of selectors to use in TCP server. */
private int selectorsCnt = DFLT_SELECTORS_CNT;
+ /**
+ * Defines how many non-blocking {@code selector.selectNow()} should be made before
+ * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+ * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+ */
+ private long selectorSpins = IgniteSystemProperties.getLong("IGNITE_SELECTOR_SPINS", 0L);
+
/** Address resolver. */
private AddressResolver addrRslvr;
@@ -863,11 +1060,17 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
};
/** Client connect futures. */
- private final ConcurrentMap<UUID, GridFutureAdapter<GridCommunicationClient>> clientFuts =
+ private final ConcurrentMap<ConnectionKey, GridFutureAdapter<GridCommunicationClient>> clientFuts =
GridConcurrentFactory.newMap();
/** */
- private final ConcurrentMap<ClientKey, GridNioRecoveryDescriptor> recoveryDescs = GridConcurrentFactory.newMap();
+ private final ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> recoveryDescs = GridConcurrentFactory.newMap();
+
+ /** */
+ private final ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> outRecDescs = GridConcurrentFactory.newMap();
+
+ /** */
+ private final ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> inRecDescs = GridConcurrentFactory.newMap();
/** Discovery listener. */
private final GridLocalEventListener discoLsnr = new GridLocalEventListener() {
@@ -976,6 +1179,49 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
return locPortRange;
}
+ /** {@inheritDoc} */
+ @Override public boolean isUsePairedConnections() {
+ return usePairedConnections;
+ }
+
+ /**
+ * Set this to {@code true} if {@code TcpCommunicationSpi} should
+ * maintain connection for outgoing and incoming messages separately.
+ * In this case total number of connections between local and each remote node
+ * is {@link #getConnectionsPerNode()} * 2.
+ * <p>
+ * Set this to {@code false} if each connection of {@link #getConnectionsPerNode()}
+ * should be used for outgoing and incoming messages. In this case total number
+ * of connections between local and each remote node is {@link #getConnectionsPerNode()}.
+ * In this case load NIO selectors load
+ * balancing of {@link GridNioServer} will be disabled.
+ * <p>
+ * Default is {@code true}.
+ *
+ * @param usePairedConnections {@code true} to use paired connections and {@code false} otherwise.
+ * @see #getConnectionsPerNode()
+ */
+ public void setUsePairedConnections(boolean usePairedConnections) {
+ this.usePairedConnections = usePairedConnections;
+ }
+
+ /**
+ * Sets number of connections to each remote node. if {@link #isUsePairedConnections()}
+ * is {@code true} then number of connections is doubled and half is used for incoming and
+ * half for outgoing messages.
+ *
+ * @param maxConnectionsPerNode Number of connections per node.
+ * @see #isUsePairedConnections()
+ */
+ public void setConnectionsPerNode(int maxConnectionsPerNode) {
+ this.connectionsPerNode = maxConnectionsPerNode;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getConnectionsPerNode() {
+ return connectionsPerNode;
+ }
+
/**
* Sets local port to accept shared memory connections.
* <p>
@@ -1222,6 +1468,22 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
return selectorsCnt;
}
+ /** {@inheritDoc} */
+ @Override public long getSelectorSpins() {
+ return selectorSpins;
+ }
+
+ /**
+ * Defines how many non-blocking {@code selector.selectNow()} should be made before
+ * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+ * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+ *
+ * @param selectorSpins Selector thread busy-loop iterations.
+ */
+ public void setSelectorSpins(long selectorSpins) {
+ this.selectorSpins = selectorSpins;
+ }
+
/**
* Sets value for {@code TCP_NODELAY} socket option. Each
* socket will be opened using provided value.
@@ -1396,7 +1658,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
if (log != null) {
StringBuilder sb = new StringBuilder("Communication SPI recovery descriptors: ").append(U.nl());
- for (Map.Entry<ClientKey, GridNioRecoveryDescriptor> entry : recoveryDescs.entrySet()) {
+ for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> entry : recoveryDescs.entrySet()) {
GridNioRecoveryDescriptor desc = entry.getValue();
sb.append(" [key=").append(entry.getKey())
@@ -1409,14 +1671,48 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
.append(']').append(U.nl());
}
- sb.append("Communication SPI clients: ").append(U.nl());
+ for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> entry : outRecDescs.entrySet()) {
+ GridNioRecoveryDescriptor desc = entry.getValue();
+
+ sb.append(" [key=").append(entry.getKey())
+ .append(", msgsSent=").append(desc.sent())
+ .append(", msgsAckedByRmt=").append(desc.acked())
+ .append(", reserveCnt=").append(desc.reserveCount())
+ .append(", connected=").append(desc.connected())
+ .append(", reserved=").append(desc.reserved())
+ .append(", descIdHash=").append(System.identityHashCode(desc))
+ .append(']').append(U.nl());
+ }
+
+ for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> entry : inRecDescs.entrySet()) {
+ GridNioRecoveryDescriptor desc = entry.getValue();
- for (Map.Entry<UUID, GridCommunicationClient> entry : clients.entrySet()) {
- sb.append(" [node=").append(entry.getKey())
- .append(", client=").append(entry.getValue())
+ sb.append(" [key=").append(entry.getKey())
+ .append(", msgsRcvd=").append(desc.received())
+ .append(", lastAcked=").append(desc.lastAcknowledged())
+ .append(", reserveCnt=").append(desc.reserveCount())
+ .append(", connected=").append(desc.connected())
+ .append(", reserved=").append(desc.reserved())
+ .append(", handshakeIdx=").append(desc.handshakeIndex())
+ .append(", descIdHash=").append(System.identityHashCode(desc))
.append(']').append(U.nl());
}
+ sb.append("Communication SPI clients: ").append(U.nl());
+
+ for (Map.Entry<UUID, GridCommunicationClient[]> entry : clients.entrySet()) {
+ UUID nodeId = entry.getKey();
+ GridCommunicationClient[] clients0 = entry.getValue();
+
+ for (GridCommunicationClient client : clients0) {
+ if (client != null) {
+ sb.append(" [node=").append(nodeId)
+ .append(", client=").append(client)
+ .append(']').append(U.nl());
+ }
+ }
+ }
+
U.warn(log, sb.toString());
}
@@ -1426,6 +1722,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
nioSrvr.dumpStats();
}
+ /** */
+ private final ThreadLocal<Integer> threadConnIdx = new ThreadLocal<>();
+
+ /** */
+ private final AtomicInteger connIdx = new AtomicInteger();
+
/** {@inheritDoc} */
@Override public Map<String, Object> getNodeAttributes() throws IgniteSpiException {
initFailureDetectionTimeout();
@@ -1439,6 +1741,8 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
assertParameter(msgQueueLimit >= 0, "msgQueueLimit >= 0");
assertParameter(shmemPort > 0 || shmemPort == -1, "shmemPort > 0 || shmemPort == -1");
assertParameter(selectorsCnt > 0, "selectorsCnt > 0");
+ assertParameter(connectionsPerNode > 0, "connectionsPerNode > 0");
+ assertParameter(connectionsPerNode <= 1024, "connectionsPerNode <= 1024");
if (!failureDetectionTimeoutEnabled()) {
assertParameter(reconCnt > 0, "reconnectCnt > 0");
@@ -1458,6 +1762,21 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
"Specified 'unackedMsgsBufSize' is too low, it should be at least 'ackSndThreshold * 5'.");
}
+ if (connectionsPerNode > 1) {
+ connPlc = new ConnectionPolicy() {
+ @Override public int connectionIndex() {
+ return (int)(U.safeAbs(Thread.currentThread().getId()) % connectionsPerNode);
+ }
+ };
+ }
+ else {
+ connPlc = new ConnectionPolicy() {
+ @Override public int connectionIndex() {
+ return 0;
+ }
+ };
+ }
+
try {
locHost = U.resolveLocalHost(locAddr);
}
@@ -1495,6 +1814,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
res.put(createSpiAttributeName(ATTR_PORT), boundTcpPort);
res.put(createSpiAttributeName(ATTR_SHMEM_PORT), boundTcpShmemPort >= 0 ? boundTcpShmemPort : null);
res.put(createSpiAttributeName(ATTR_EXT_ADDRS), extAddrs);
+ res.put(createSpiAttributeName(ATTR_PAIRED_CONN), usePairedConnections);
return res;
}
@@ -1524,6 +1844,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
log.debug(configInfo("sockRcvBuf", sockRcvBuf));
log.debug(configInfo("shmemPort", shmemPort));
log.debug(configInfo("msgQueueLimit", msgQueueLimit));
+ log.debug(configInfo("connectionsPerNode", connectionsPerNode));
if (failureDetectionTimeoutEnabled()) {
log.debug(configInfo("connTimeout", connTimeout));
@@ -1548,6 +1869,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
", slowClientQueueLimit=" + slowClientQueueLimit + ']');
}
+ if (msgQueueLimit == 0)
+ U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " +
+ "potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " +
+ "due to message queues growth on sender and reciever sides.");
+
registerMBean(gridName, this, TcpCommunicationSpiMBean.class);
connectGate = new ConnectGateway();
@@ -1642,9 +1968,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
assert formatter != null;
- UUID rmtNodeId = ses.meta(NODE_ID_META);
+ ConnectionKey key = ses.meta(CONN_IDX_META);
- return rmtNodeId != null ? formatter.reader(rmtNodeId, msgFactory) : null;
+ return key != null ? formatter.reader(key.nodeId(), msgFactory) : null;
}
};
@@ -1657,9 +1983,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
assert formatter != null;
- UUID rmtNodeId = ses.meta(NODE_ID_META);
+ ConnectionKey key = ses.meta(CONN_IDX_META);
- return rmtNodeId != null ? formatter.writer(rmtNodeId) : null;
+ return key != null ? formatter.writer(key.nodeId()) : null;
}
};
@@ -1716,6 +2042,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
.logger(log)
.selectorCount(selectorsCnt)
.gridName(gridName)
+ .serverName("tcp-comm")
.tcpNoDelay(tcpNoDelay)
.directBuffer(directBuf)
.byteOrder(ByteOrder.nativeOrder())
@@ -1725,18 +2052,21 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
.directMode(true)
.metricsListener(metricsLsnr)
.writeTimeout(sockWriteTimeout)
+ .selectorSpins(selectorSpins)
.filters(filters)
.writerFactory(writerFactory)
.skipRecoveryPredicate(skipRecoveryPred)
.messageQueueSizeListener(queueSizeMonitor)
+ .balancing(usePairedConnections) // Current balancing logic assumes separate in/out connections.
.build();
boundTcpPort = port;
// Ack Port the TCP server was bound to.
if (log.isInfoEnabled())
- log.info("Successfully bound to TCP port [port=" + boundTcpPort +
- ", locHost=" + locHost + ']');
+ log.info("Successfully bound communication NIO server to TCP port " +
+ "[port=" + boundTcpPort + ", locHost=" + locHost + ", selectorsCnt=" + selectorsCnt +
+ ", selectorSpins=" + srvr.selectorSpins() + ']');
srvr.idleTimeout(idleConnTimeout);
@@ -1837,8 +2167,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
shmemWorkers.clear();
// Force closing on stop (safety).
- for (GridCommunicationClient client : clients.values())
- client.forceClose();
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null)
+ client.forceClose();
+ }
+ }
// Clear resources.
nioSrvr = null;
@@ -1863,8 +2197,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
connectGate.stopped();
// Force closing.
- for (GridCommunicationClient client : clients.values())
- client.forceClose();
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null)
+ client.forceClose();
+ }
+ }
getSpiContext().deregisterPorts();
@@ -1875,8 +2213,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
@Override public void onClientDisconnected(IgniteFuture<?> reconnectFut) {
connectGate.disconnected(reconnectFut);
- for (GridCommunicationClient client : clients.values())
- client.forceClose();
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null)
+ client.forceClose();
+ }
+ }
IgniteClientDisconnectedCheckedException err = new IgniteClientDisconnectedCheckedException(reconnectFut,
"Failed to connect client node disconnected.");
@@ -1885,6 +2227,8 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
clientFut.onDone(err);
recoveryDescs.clear();
+ inRecDescs.clear();
+ outRecDescs.clear();
}
/** {@inheritDoc} */
@@ -1898,16 +2242,18 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
void onNodeLeft(UUID nodeId) {
assert nodeId != null;
- GridCommunicationClient client = clients.get(nodeId);
-
- if (client != null) {
- if (log.isDebugEnabled())
- log.debug("Forcing NIO client close since node has left [nodeId=" + nodeId +
- ", client=" + client + ']');
+ GridCommunicationClient[] clients0 = clients.remove(nodeId);
- client.forceClose();
+ if (clients0 != null) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null) {
+ if (log.isDebugEnabled())
+ log.debug("Forcing NIO client close since node has left [nodeId=" + nodeId +
+ ", client=" + client + ']');
- clients.remove(nodeId, client);
+ client.forceClose();
+ }
+ }
}
}
@@ -1982,11 +2328,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
else {
GridCommunicationClient client = null;
+ int connIdx = useMultipleConnections(node) ? connPlc.connectionIndex() : 0;
+
try {
boolean retry;
do {
- client = reserveClient(node);
+ client = reserveClient(node, connIdx);
UUID nodeId = null;
@@ -2000,7 +2348,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
if (!retry)
sentMsgsCnt.increment();
else {
- clients.remove(node.id(), client);
+ removeNodeClient(node.id(), client);
ClusterNode node0 = getSpiContext().node(node.id());
@@ -2017,26 +2365,94 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
throw new IgniteSpiException("Failed to send message to remote node: " + node, e);
}
finally {
- if (client != null && clients.remove(node.id(), client))
+ if (client != null && removeNodeClient(node.id(), client))
client.forceClose();
}
}
}
/**
+ * @param nodeId Node ID.
+ * @param rmvClient Client to remove.
+ * @return {@code True} if client was removed.
+ */
+ private boolean removeNodeClient(UUID nodeId, GridCommunicationClient rmvClient) {
+ for (;;) {
+ GridCommunicationClient[] curClients = clients.get(nodeId);
+
+ if (curClients == null || rmvClient.connectionIndex() >= curClients.length || curClients[rmvClient.connectionIndex()] != rmvClient)
+ return false;
+
+ GridCommunicationClient[] newClients = Arrays.copyOf(curClients, curClients.length);
+
+ newClients[rmvClient.connectionIndex()] = null;
+
+ if (clients.replace(nodeId, curClients, newClients))
+ return true;
+ }
+ }
+
+ /**
+ * @param node Node.
+ * @param connIdx Connection index.
+ * @param addClient Client to add.
+ */
+ private void addNodeClient(ClusterNode node, int connIdx, GridCommunicationClient addClient) {
+ assert connectionsPerNode > 0 : connectionsPerNode;
+ assert connIdx == addClient.connectionIndex() : addClient;
+
+ if (connIdx >= connectionsPerNode) {
+ assert !usePairedConnections(node);
+
+ return;
+ }
+
+ for (;;) {
+ GridCommunicationClient[] curClients = clients.get(node.id());
+
+ assert curClients == null || curClients[connIdx] == null : "Client already created [node=" + node.id() +
+ ", connIdx=" + connIdx +
+ ", client=" + addClient +
+ ", oldClient=" + curClients[connIdx] + ']';
+
+ GridCommunicationClient[] newClients;
+
+ if (curClients == null) {
+ newClients = new GridCommunicationClient[useMultipleConnections(node) ? connectionsPerNode : 1];
+ newClients[connIdx] = addClient;
+
+ if (clients.putIfAbsent(node.id(), newClients) == null)
+ break;
+ }
+ else {
+ newClients = Arrays.copyOf(curClients, curClients.length);
+ newClients[connIdx] = addClient;
+
+ if (clients.replace(node.id(), curClients, newClients))
+ break;
+ }
+ }
+ }
+
+ /**
* Returns existing or just created client to node.
*
* @param node Node to which client should be open.
+ * @param connIdx Connection index.
* @return The existing or just created client.
* @throws IgniteCheckedException Thrown if any exception occurs.
*/
- private GridCommunicationClient reserveClient(ClusterNode node) throws IgniteCheckedException {
+ private GridCommunicationClient reserveClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
assert node != null;
+ assert (connIdx >= 0 && connIdx < connectionsPerNode) || !usePairedConnections(node) : connIdx;
UUID nodeId = node.id();
while (true) {
- GridCommunicationClient client = clients.get(nodeId);
+ GridCommunicationClient[] curClients = clients.get(nodeId);
+
+ GridCommunicationClient client = curClients != null && connIdx < curClients.length ?
+ curClients[connIdx] : null;
if (client == null) {
if (stopping)
@@ -2045,25 +2461,27 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
// Do not allow concurrent connects.
GridFutureAdapter<GridCommunicationClient> fut = new ConnectFuture();
- GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(nodeId, fut);
+ ConnectionKey connKey = new ConnectionKey(nodeId, connIdx, -1);
+
+ GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(connKey, fut);
if (oldFut == null) {
try {
- GridCommunicationClient client0 = clients.get(nodeId);
+ GridCommunicationClient[] curClients0 = clients.get(nodeId);
+
+ GridCommunicationClient client0 = curClients0 != null && connIdx < curClients0.length ?
+ curClients0[connIdx] : null;
if (client0 == null) {
- client0 = createNioClient(node);
+ client0 = createNioClient(node, connIdx);
if (client0 != null) {
- GridCommunicationClient old = clients.put(nodeId, client0);
-
- assert old == null : "Client already created " +
- "[node=" + node + ", client=" + client0 + ", oldClient=" + old + ']';
+ addNodeClient(node, connIdx, client0);
if (client0 instanceof GridTcpNioCommunicationClient) {
GridTcpNioCommunicationClient tcpClient = ((GridTcpNioCommunicationClient)client0);
- if (tcpClient.session().closeTime() > 0 && clients.remove(nodeId, client0)) {
+ if (tcpClient.session().closeTime() > 0 && removeNodeClient(nodeId, client0)) {
if (log.isDebugEnabled())
log.debug("Session was closed after client creation, will retry " +
"[node=" + node + ", client=" + client0 + ']');
@@ -2085,7 +2503,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
throw (Error)e;
}
finally {
- clientFuts.remove(nodeId, fut);
+ clientFuts.remove(connKey, fut);
}
}
else
@@ -2097,27 +2515,31 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
continue;
if (getSpiContext().node(nodeId) == null) {
- if (clients.remove(nodeId, client))
+ if (removeNodeClient(nodeId, client))
client.forceClose();
throw new IgniteSpiException("Destination node is not in topology: " + node.id());
}
}
+ assert connIdx == client.connectionIndex() : client;
+
if (client.reserve())
return client;
else
// Client has just been closed by idle worker. Help it and try again.
- clients.remove(nodeId, client);
+ removeNodeClient(nodeId, client);
}
}
/**
* @param node Node to create client for.
+ * @param connIdx Connection index.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
- @Nullable protected GridCommunicationClient createNioClient(ClusterNode node) throws IgniteCheckedException {
+ @Nullable private GridCommunicationClient createNioClient(ClusterNode node, int connIdx)
+ throws IgniteCheckedException {
assert node != null;
Integer shmemPort = node.attribute(createSpiAttributeName(ATTR_SHMEM_PORT));
@@ -2136,6 +2558,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
try {
GridCommunicationClient client = createShmemClient(
node,
+ connIdx,
shmemPort);
if (log.isDebugEnabled())
@@ -2158,7 +2581,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
connectGate.enter();
try {
- GridCommunicationClient client = createTcpClient(node);
+ GridCommunicationClient client = createTcpClient(node, connIdx);
if (log.isDebugEnabled())
log.debug("TCP client created: " + client);
@@ -2173,10 +2596,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
/**
* @param node Node.
* @param port Port.
+ * @param connIdx Connection index.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
- @Nullable protected GridCommunicationClient createShmemClient(ClusterNode node,
+ @Nullable private GridCommunicationClient createShmemClient(ClusterNode node,
+ int connIdx,
Integer port) throws IgniteCheckedException {
int attempt = 1;
@@ -2190,7 +2615,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
GridCommunicationClient client;
try {
- client = new GridShmemCommunicationClient(metricsLsnr,
+ client = new GridShmemCommunicationClient(
+ connIdx,
+ metricsLsnr,
port,
timeoutHelper.nextTimeoutChunk(connTimeout),
log,
@@ -2211,7 +2638,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
}
try {
- safeHandshake(client, null, node.id(), timeoutHelper.nextTimeoutChunk(connTimeout0), null);
+ safeHandshake(client,
+ null,
+ node.id(),
+ timeoutHelper.nextTimeoutChunk(connTimeout0),
+ null,
+ null);
}
catch (HandshakeTimeoutException | IgniteSpiOperationTimeoutException e) {
client.forceClose();
@@ -2270,10 +2702,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
*/
private void checkClientQueueSize(GridNioSession ses, int msgQueueSize) {
if (slowClientQueueLimit > 0 && msgQueueSize > slowClientQueueLimit) {
- UUID id = ses.meta(NODE_ID_META);
+ ConnectionKey id = ses.meta(CONN_IDX_META);
if (id != null) {
- ClusterNode node = getSpiContext().node(id);
+ ClusterNode node = getSpiContext().node(id.nodeId);
if (node != null && node.isClient()) {
String msg = "Client node outbound message queue size exceeded slowClientQueueLimit, " +
@@ -2283,11 +2715,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
", clientNode=" + node +
", slowClientQueueLimit=" + slowClientQueueLimit + ']';
- U.quietAndWarn(
- log,
- msg);
+ U.quietAndWarn(log, msg);
- getSpiContext().failNode(id, msg);
+ getSpiContext().failNode(id.nodeId(), msg);
}
}
}
@@ -2297,10 +2727,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
* Establish TCP connection to remote node and returns client.
*
* @param node Remote node.
+ * @param connIdx Connection index.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
- protected GridCommunicationClient createTcpClient(ClusterNode node) throws IgniteCheckedException {
+ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
Collection<String> rmtAddrs0 = node.attribute(createSpiAttributeName(ATTR_ADDRS));
Collection<String> rmtHostNames0 = node.attribute(createSpiAttributeName(ATTR_HOST_NAMES));
Integer boundPort = node.attribute(createSpiAttributeName(ATTR_PORT));
@@ -2368,7 +2799,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
"(node left topology): " + node);
}
- GridNioRecoveryDescriptor recoveryDesc = recoveryDescriptor(node);
+ ConnectionKey connKey = new ConnectionKey(node.id(), connIdx, -1);
+
+ GridNioRecoveryDescriptor recoveryDesc = outRecoveryDescriptor(node, connKey);
if (!recoveryDesc.reserve()) {
U.closeQuiet(ch);
@@ -2395,11 +2828,14 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
sslMeta.sslEngine(sslEngine);
}
+ Integer handshakeConnIdx = useMultipleConnections(node) ? connIdx : null;
+
rcvCnt = safeHandshake(ch,
recoveryDesc,
node.id(),
timeoutHelper.nextTimeoutChunk(connTimeout0),
- sslMeta);
+ sslMeta,
+ handshakeConnIdx);
if (rcvCnt == -1)
return null;
@@ -2410,7 +2846,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
}
try {
- meta.put(NODE_ID_META, node.id());
+ meta.put(CONN_IDX_META, connKey);
if (recoveryDesc != null) {
recoveryDesc.onHandshake(rcvCnt);
@@ -2420,7 +2856,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
GridNioSession ses = nioSrvr.createSession(ch, meta).get();
- client = new GridTcpNioCommunicationClient(ses, log);
+ client = new GridTcpNioCommunicationClient(connIdx, ses, log);
conn = true;
}
@@ -2564,6 +3000,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
* @param rmtNodeId Remote node.
* @param timeout Timeout for handshake.
* @param sslMeta Session meta.
+ * @param handshakeConnIdx Non null connection index if need send it in handshake.
* @throws IgniteCheckedException If handshake failed or wasn't completed withing timeout.
* @return Handshake response.
*/
@@ -2573,7 +3010,8 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
@Nullable GridNioRecoveryDescriptor recovery,
UUID rmtNodeId,
long timeout,
- GridSslMeta sslMeta
+ GridSslMeta sslMeta,
+ @Nullable Integer handshakeConnIdx
) throws IgniteCheckedException {
HandshakeTimeoutObject<T> obj = new HandshakeTimeoutObject<>(client, U.currentTimeMillis() + timeout);
@@ -2655,14 +3093,28 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
"fully initialized [isStopping=" + getSpiContext().isStopping() + ']');
if (recovery != null) {
- HandshakeMessage msg = new HandshakeMessage(locNode.id(),
- recovery.incrementConnectCount(),
- recovery.received());
+ HandshakeMessage msg;
+
+ int msgSize = 33;
+
+ if (handshakeConnIdx != null) {
+ msg = new HandshakeMessage2(locNode.id(),
+ recovery.incrementConnectCount(),
+ recovery.received(),
+ handshakeConnIdx);
+
+ msgSize += 4;
+ }
+ else {
+ msg = new HandshakeMessage(locNode.id(),
+ recovery.incrementConnectCount(),
+ recovery.received());
+ }
if (log.isDebugEnabled())
log.debug("Write handshake message [rmtNode=" + rmtNodeId + ", msg=" + msg + ']');
- buf = ByteBuffer.allocate(33);
+ buf = ByteBuffer.allocate(msgSize);
buf.order(ByteOrder.nativeOrder());
@@ -2689,6 +3141,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
else
ch.write(ByteBuffer.wrap(nodeIdMessage().nodeIdBytesWithType));
}
+
if (recovery != null) {
if (log.isDebugEnabled())
log.debug("Waiting for handshake [rmtNode=" + rmtNodeId + ']');
@@ -2818,26 +3271,81 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
U.join(commWorker, log);
- for (GridCommunicationClient client : clients.values())
- client.forceClose();
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null)
+ client.forceClose();
+ }
+ }
+ }
+
+ /**
+ * @param node Node.
+ * @param key Connection key.
+ * @return Recovery descriptor for outgoing connection.
+ */
+ private GridNioRecoveryDescriptor outRecoveryDescriptor(ClusterNode node, ConnectionKey key) {
+ if (usePairedConnections(node))
+ return recoveryDescriptor(outRecDescs, true, node, key);
+ else
+ return recoveryDescriptor(recoveryDescs, false, node, key);
}
/**
* @param node Node.
- * @return Recovery receive data for given node.
+ * @param key Connection key.
+ * @return Recovery descriptor for incoming connection.
+ */
+ private GridNioRecoveryDescriptor inRecoveryDescriptor(ClusterNode node, ConnectionKey key) {
+ if (usePairedConnections(node))
+ return recoveryDescriptor(inRecDescs, true, node, key);
+ else
+ return recoveryDescriptor(recoveryDescs, false, node, key);
+ }
+
+ /**
+ * @param node Node.
+ * @return {@code True} if given node supports multiple connections per-node for communication.
+ */
+ private boolean useMultipleConnections(ClusterNode node) {
+ return node.version().compareToIgnoreTimestamp(MULTIPLE_CONN_SINCE_VER) >= 0;
+ }
+
+ /**
+ * @param node Node.
+ * @return {@code True} if can use in/out connection pair for communication.
*/
- private GridNioRecoveryDescriptor recoveryDescriptor(ClusterNode node) {
- ClientKey id = new ClientKey(node.id(), node.order());
+ private boolean usePairedConnections(ClusterNode node) {
+ if (usePairedConnections) {
+ Boolean attr = node.attribute(createSpiAttributeName(ATTR_PAIRED_CONN));
+
+ return attr != null && attr;
+ }
+
+ return false;
+ }
- GridNioRecoveryDescriptor recovery = recoveryDescs.get(id);
+ /**
+ * @param recoveryDescs Descriptors map.
+ * @param pairedConnections {@code True} if in/out connections pair is used for communication with node.
+ * @param node Node.
+ * @param key Connection key.
+ * @return Recovery receive data for given node.
+ */
+ private GridNioRecoveryDescriptor recoveryDescriptor(
+ ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> recoveryDescs,
+ boolean pairedConnections,
+ ClusterNode node,
+ ConnectionKey key) {
+ GridNioRecoveryDescriptor recovery = recoveryDescs.get(key);
if (recovery == null) {
int maxSize = Math.max(msgQueueLimit, ackSndThreshold);
- int queueLimit = unackedMsgsBufSize != 0 ? unackedMsgsBufSize : (maxSize * 5);
+ int queueLimit = unackedMsgsBufSize != 0 ? unackedMsgsBufSize : (maxSize * 128);
- GridNioRecoveryDescriptor old =
- recoveryDescs.putIfAbsent(id, recovery = new GridNioRecoveryDescriptor(queueLimit, node, log));
+ GridNioRecoveryDescriptor old = recoveryDescs.putIfAbsent(key,
+ recovery = new GridNioRecoveryDescriptor(pairedConnections, queueLimit, node, log));
if (old != null)
recovery = old;
@@ -2879,54 +3387,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
return S.toString(TcpCommunicationSpi.class, this);
}
- /**
- *
- */
- private static class ClientKey {
- /** */
- private UUID nodeId;
-
- /** */
- private long order;
-
- /**
- * @param nodeId Node ID.
- * @param order Node order.
- */
- private ClientKey(UUID nodeId, long order) {
- this.nodeId = nodeId;
- this.order = order;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- if (this == obj)
- return true;
-
- if (obj == null || getClass() != obj.getClass())
- return false;
-
- ClientKey other = (ClientKey)obj;
-
- return order == other.order && nodeId.equals(other.nodeId);
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = nodeId.hashCode();
-
- res = 31 * res + (int)(order ^ (order >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(ClientKey.class, this);
- }
- }
-
/** Internal exception class for proper timeout handling. */
private static class HandshakeTimeoutException extends IgniteCheckedException {
/** */
@@ -3026,9 +3486,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
assert formatter != null;
- UUID rmtNodeId = ses.meta(NODE_ID_META);
+ ConnectionKey connKey = ses.meta(CONN_IDX_META);
- return rmtNodeId != null ? formatter.writer(rmtNodeId) : null;
+ return connKey != null ? formatter.writer(connKey.nodeId()) : null;
}
};
@@ -3042,9 +3502,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
assert formatter != null;
- UUID rmtNodeId = ses.meta(NODE_ID_META);
+ ConnectionKey connKey = ses.meta(CONN_IDX_META);
- return rmtNodeId != null ? formatter.reader(rmtNodeId, msgFactory) : null;
+ return connKey != null ? formatter.reader(connKey.nodeId(), msgFactory) : null;
}
};
@@ -3125,62 +3585,108 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
private void processIdle() {
cleanupRecovery();
- for (Map.Entry<UUID, GridCommunicationClient> e : clients.entrySet()) {
+ for (Map.Entry<UUID, GridCommunicationClient[]> e : clients.entrySet()) {
UUID nodeId = e.getKey();
- GridCommunicationClient client = e.getValue();
+ for (GridCommunicationClient client : e.getValue()) {
+ if (client == null)
+ continue;
- ClusterNode node = getSpiContext().node(nodeId);
+ ClusterNode node = getSpiContext().node(nodeId);
- if (node == null) {
- if (log.isDebugEnabled())
- log.debug("Forcing close of non-existent node connection: " + nodeId);
+ if (node == null) {
+ if (log.isDebugEnabled())
+ log.debug("Forcing close of non-existent node connection: " + nodeId);
- client.forceClose();
+ client.forceClose();
- clients.remove(nodeId, client);
+ removeNodeClient(nodeId, client);
- continue;
- }
+ continue;
+ }
- GridNioRecoveryDescriptor recovery = null;
+ GridNioRecoveryDescriptor recovery = null;
- if (client instanceof GridTcpNioCommunicationClient) {
- recovery = recoveryDescs.get(new ClientKey(node.id(), node.order()));
+ if (!usePairedConnections(node) && client instanceof GridTcpNioCommunicationClient) {
+ recovery = recoveryDescs.get(new ConnectionKey(node.id(), client.connectionIndex(), -1));
- if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
- RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
+ if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
+ RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
- if (log.isDebugEnabled())
- log.debug("Send recovery acknowledgement on timeout [rmtNode=" + nodeId +
- ", rcvCnt=" + msg.received() + ']');
+ if (log.isDebugEnabled())
+ log.debug("Send recovery acknowledgement on timeout [rmtNode=" + nodeId +
+ ", rcvCnt=" + msg.received() + ']');
- nioSrvr.sendSystem(((GridTcpNioCommunicationClient)client).session(), msg);
+ try {
+ nioSrvr.sendSystem(((GridTcpNioCommunicationClient)client).session(), msg);
- recovery.lastAcknowledged(msg.received());
+ recovery.lastAcknowledged(msg.received());
+ }
+ catch (IgniteCheckedException err) {
+ U.error(log, "Failed to send message: " + err, err);
+ }
- continue;
+ continue;
+ }
}
- }
- long idleTime = client.getIdleTime();
+ long idleTime = client.getIdleTime();
+
+ if (idleTime >= idleConnTimeout) {
+ if (recovery == null && usePairedConnections(node))
+ recovery = outRecDescs.get(new ConnectionKey(node.id(), client.connectionIndex(), -1));
+
+ if (recovery != null &&
+ recovery.nodeAlive(getSpiContext().node(nodeId)) &&
+ !recovery.messagesRequests().isEmpty()) {
+ if (log.isDebugEnabled())
+ log.debug("Node connection is idle, but there are unacknowledged messages, " +
+ "will wait: " + nodeId);
+
+ continue;
+ }
- if (idleTime >= idleConnTimeout) {
- if (recovery != null &&
- recovery.nodeAlive(getSpiContext().node(nodeId)) &&
- !recovery.messagesFutures().isEmpty()) {
if (log.isDebugEnabled())
- log.debug("Node connection is idle, but there are unacknowledged messages, " +
- "will wait: " + nodeId);
+ log.debug("Closing idle node connection: " + nodeId);
- continue;
+ if (client.close() || client.closed())
+ removeNodeClient(nodeId, client);
}
+ }
+ }
- if (log.isDebugEnabled())
- log.debug("Closing idle node connection: " + nodeId);
+ for (GridNioSession ses : nioSrvr.sessions()) {
+ GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
+
+ if (recovery != null && usePairedConnections(recovery.node())) {
+ assert ses.accepted() : ses;
+
+ sendAckOnTimeout(recovery, ses);
+ }
+ }
+ }
+
+ /**
+ * @param recovery Recovery descriptor.
+ * @param ses Session.
+ */
+ private void sendAckOnTimeout(GridNioRecoveryDescriptor recovery, GridNioSession ses) {
+ if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
+ RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
+
+ if (log.isDebugEnabled()) {
+ log.debug("Send recovery acknowledgement on timeout [rmtNode=" + recovery.node().id() +
+ ", rcvCnt=" + msg.received() +
+ ", lastAcked=" + recovery.lastAcknowledged() + ']');
+ }
+
+ try {
+ nioSrvr.sendSystem(ses, msg);
- if (client.close() || client.closed())
- clients.remove(nodeId, client);
+ recovery.lastAcknowledged(msg.received());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send message: " + e, e);
}
}
}
@@ -3189,15 +3695,24 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
*
*/
private void cleanupRecovery() {
- Set<ClientKey> left = null;
+ cleanupRecovery(recoveryDescs);
+ cleanupRecovery(inRecDescs);
+ cleanupRecovery(outRecDescs);
+ }
+
+ /**
+ * @param recoveryDescs Recovery descriptors to cleanup.
+ */
+ private void cleanupRecovery(ConcurrentMap<ConnectionKey, GridNioRecoveryDescriptor> recoveryDescs) {
+ Set<ConnectionKey> left = null;
- for (Map.Entry<ClientKey, GridNioRecoveryDescriptor> e : recoveryDescs.entrySet()) {
+ for (Map.Entry<ConnectionKey, GridNioRecoveryDescriptor> e : recoveryDescs.entrySet()) {
if (left != null && left.contains(e.getKey()))
continue;
- GridNioRecoveryDescriptor recoverySnd = e.getValue();
+ GridNioRecoveryDescriptor recoveryDesc = e.getValue();
- if (!recoverySnd.nodeAlive(getSpiContext().node(recoverySnd.node().id()))) {
+ if (!recoveryDesc.nodeAlive(getSpiContext().node(e.getKey().nodeId()))) {
if (left == null)
left = new HashSet<>();
@@ -3208,11 +3723,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
if (left != null) {
assert !left.isEmpty();
- for (ClientKey id : left) {
- GridNioRecoveryDescriptor recoverySnd = recoveryDescs.get(id);
+ for (ConnectionKey id : left) {
+ GridNioRecoveryDescriptor recoveryDesc = recoveryDescs.get(id);
- if (recoverySnd != null && recoverySnd.onNodeLeft())
- recoveryDescs.remove(id);
+ if (recoveryDesc != null && recoveryDesc.onNodeLeft())
+ recoveryDescs.remove(id, recoveryDesc);
}
}
}
@@ -3221,45 +3736,43 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
* @param sesInfo Disconnected session information.
*/
private void processDisconnect(DisconnectedSessionInfo sesInfo) {
- if (sesInfo.reconnect) {
- GridNioRecoveryDescriptor recoveryDesc = sesInfo.recoveryDesc;
-
- ClusterNode node = recoveryDesc.node();
+ GridNioRecoveryDescriptor recoveryDesc = sesInfo.recoveryDesc;
- if (!recoveryDesc.nodeAlive(getSpiContext().node(node.id())))
- return;
+ ClusterNode node = recoveryDesc.node();
- try {
- if (log.isDebugEnabled())
- log.debug("Recovery reconnect [rmtNode=" + recoveryDesc.node().id() + ']');
+ if (!recoveryDesc.nodeAlive(getSpiContext().node(node.id())))
+ return;
- GridCommunicationClient client = reserveClient(node);
+ try {
+ if (log.isDebugEnabled())
+ log.debug("Recovery reconnect [rmtNode=" + recoveryDesc.node().id() + ']');
- client.release();
- }
- catch (IgniteCheckedException | IgniteException e) {
- try {
- if (recoveryDesc.nodeAlive(getSpiContext().node(node.id())) && getSpiContext().pingNode(node.id())) {
- if (log.isDebugEnabled())
- log.debug("Recovery reconnect failed, will retry " +
- "[rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
+ GridCommunicationClient client = reserveClient(node, sesInfo.connIdx);
- addProcessDisconnectRequest(sesInfo);
- }
- else {
- if (log.isDebugEnabled())
- log.debug("Recovery reconnect failed, " +
- "node left [rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
+ client.release();
+ }
+ catch (IgniteCheckedException | IgniteException e) {
+ try {
+ if (recoveryDesc.nodeAlive(getSpiContext().node(node.id())) && getSpiContext().pingNode(node.id())) {
+ if (log.isDebugEnabled())
+ log.debug("Recovery reconnect failed, will retry " +
+ "[rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
- onException("Recovery reconnect failed, node left [rmtNode=" + recoveryDesc.node().id() + "]",
- e);
- }
+ addProcessDisconnectRequest(sesInfo);
}
- catch (IgniteClientDisconnectedException e0) {
+ else {
if (log.isDebugEnabled())
- log.debug("Failed to ping node, client disconnected.");
+ log.debug("Recovery reconnect failed, " +
+ "node left [rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
+
+ onE
<TRUNCATED>
[41/50] [abbrv] ignite git commit: ignite-4371 Avoid synchronous
'rollback' call from system threads
Posted by sb...@apache.org.
ignite-4371 Avoid synchronous 'rollback' call from system threads
(cherry picked from commit 0c782b0)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c1fde05d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c1fde05d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c1fde05d
Branch: refs/heads/master
Commit: c1fde05d05251353f0e75125aca44ab145798c8e
Parents: ad785cb
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 16 19:15:48 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 19 11:43:34 2016 +0300
----------------------------------------------------------------------
.../GridCachePartitionExchangeManager.java | 37 +++---
.../processors/cache/GridCacheProcessor.java | 2 +-
.../processors/cache/GridCacheUtils.java | 10 +-
.../GridDistributedTxRemoteAdapter.java | 2 +-
.../distributed/dht/GridDhtTxFinishFuture.java | 74 +++++------
.../cache/distributed/dht/GridDhtTxLocal.java | 125 +++++++------------
.../distributed/dht/GridDhtTxLocalAdapter.java | 7 +-
.../near/GridNearTxFinishFuture.java | 44 +++----
.../cache/distributed/near/GridNearTxLocal.java | 48 ++++---
.../cache/transactions/IgniteTxHandler.java | 13 +-
.../transactions/IgniteTxLocalAdapter.java | 3 +-
.../cache/transactions/IgniteTxManager.java | 7 +-
.../GridCacheMissingCommitVersionSelfTest.java | 6 +-
.../IgniteTxStoreExceptionAbstractSelfTest.java | 1 +
14 files changed, 161 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 503b334..d26031c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -57,6 +57,7 @@ import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
@@ -854,24 +855,30 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
ready = cacheCtx.started();
if (ready) {
- GridDhtPartitionFullMap locMap = cacheCtx.topology().partitionMap(true);
+ GridAffinityAssignmentCache affCache = cacheCtx.affinity().affinityCache();
- if (useOldApi) {
- locMap = new GridDhtPartitionFullMap(locMap.nodeId(),
- locMap.nodeOrder(),
- locMap.updateSequence(),
- locMap);
- }
+ if (affCache != null) {
+ GridDhtPartitionFullMap locMap = cacheCtx.topology().partitionMap(true);
+
+ if (useOldApi) {
+ locMap = new GridDhtPartitionFullMap(locMap.nodeId(),
+ locMap.nodeOrder(),
+ locMap.updateSequence(),
+ locMap);
+ }
- addFullPartitionsMap(m,
- dupData,
- compress,
- cacheCtx.cacheId(),
- locMap,
- cacheCtx.affinity().affinityCache().similarAffinityKey());
+ addFullPartitionsMap(m,
+ dupData,
+ compress,
+ cacheCtx.cacheId(),
+ locMap,
+ affCache.similarAffinityKey());
- if (exchId != null)
- m.addPartitionUpdateCounters(cacheCtx.cacheId(), cacheCtx.topology().updateCounters(true));
+ if (exchId != null)
+ m.addPartitionUpdateCounters(cacheCtx.cacheId(), cacheCtx.topology().updateCounters(true));
+ }
+ else
+ assert cctx.cacheContext(cacheCtx.cacheId()) == null : cacheCtx.name();
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 0e0d769..cb96225 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1761,7 +1761,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* @param req Stop request.
*/
- public void prepareCacheStop(DynamicCacheChangeRequest req) {
+ private void prepareCacheStop(DynamicCacheChangeRequest req) {
assert req.stop() || req.close() : req;
GridCacheAdapter<?, ?> cache = caches.remove(maskNull(req.cacheName()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 3178203..fef6ddd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -928,9 +928,13 @@ public class GridCacheUtils {
if (tx == null)
return "null";
- return tx.getClass().getSimpleName() + "[id=" + tx.xid() + ", concurrency=" + tx.concurrency() +
- ", isolation=" + tx.isolation() + ", state=" + tx.state() + ", invalidate=" + tx.isInvalidate() +
- ", rollbackOnly=" + tx.isRollbackOnly() + ", nodeId=" + tx.nodeId() +
+ return tx.getClass().getSimpleName() + "[id=" + tx.xid() +
+ ", concurrency=" + tx.concurrency() +
+ ", isolation=" + tx.isolation() +
+ ", state=" + tx.state() +
+ ", invalidate=" + tx.isInvalidate() +
+ ", rollbackOnly=" + tx.isRollbackOnly() +
+ ", nodeId=" + tx.nodeId() +
", duration=" + (U.currentTimeMillis() - tx.startTime()) + ']';
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 4adfa8b..68c0e57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -386,7 +386,7 @@ public class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
// If another thread is doing prepare or rollback.
if (!state(PREPARING)) {
// In optimistic mode prepare may be called multiple times.
- if(state() != PREPARING || !optimistic()) {
+ if (state() != PREPARING || !optimistic()) {
if (log.isDebugEnabled())
log.debug("Invalid transaction state for prepare: " + this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index ac2ab41..147cbea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -33,8 +33,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
-import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
-import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
import org.apache.ignite.internal.util.future.GridCompoundIdentityFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -94,9 +92,6 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
/** Near mappings. */
private Map<UUID, GridDistributedTxMapping> nearMap;
- /** Trackable flag. */
- private boolean trackable = true;
-
/**
* @param cctx Context.
* @param tx Transaction.
@@ -151,46 +146,24 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
/** {@inheritDoc} */
@Override public boolean trackable() {
- return trackable;
+ return true;
}
/** {@inheritDoc} */
@Override public void markNotTrackable() {
- trackable = false;
+ assert false;
}
/**
* @param e Error.
*/
- public void onError(Throwable e) {
- if (ERR_UPD.compareAndSet(this, null, e)) {
- boolean marked = tx.setRollbackOnly();
-
- if (e instanceof IgniteTxRollbackCheckedException) {
- if (marked) {
- try {
- tx.rollback();
- }
- catch (IgniteCheckedException ex) {
- U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
- }
- }
- }
- else if (tx.isSystemInvalidate()) { // Invalidate remote transactions on heuristic error.
- finish();
+ public void rollbackOnError(Throwable e) {
+ assert e != null;
- try {
- get();
- }
- catch (IgniteTxHeuristicCheckedException ignore) {
- // Future should complete with GridCacheTxHeuristicException.
- }
- catch (IgniteCheckedException err) {
- U.error(log, "Failed to invalidate transaction: " + tx, err);
- }
- }
+ if (ERR_UPD.compareAndSet(this, null, e)) {
+ tx.setRollbackOnly();
- onComplete();
+ finish(false);
}
}
@@ -240,12 +213,21 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
/** {@inheritDoc} */
@Override public boolean onDone(IgniteInternalTx tx, Throwable err) {
if (initialized() || err != null) {
- if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING))
- this.tx.tmFinish(err == null);
-
Throwable e = this.err;
- if (e == null && commit)
+ if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) {
+ try {
+ this.tx.tmFinish(err == null);
+ }
+ catch (IgniteCheckedException finishErr) {
+ U.error(log, "Failed to finish tx: " + tx, e);
+
+ if (e == null)
+ e = finishErr;
+ }
+ }
+
+ if (commit && e == null)
e = this.tx.commitError();
Throwable finishErr = e != null ? e : err;
@@ -255,7 +237,7 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
finishErr = this.tx.commitError();
if (this.tx.syncMode() != PRIMARY_SYNC)
- this.tx.sendFinishReply(commit, finishErr);
+ this.tx.sendFinishReply(finishErr);
// Don't forget to clean up.
cctx.mvcc().removeFuture(futId);
@@ -284,13 +266,15 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
/**
* Initializes future.
+ *
+ * @param commit Commit flag.
*/
@SuppressWarnings({"SimplifiableIfStatement", "IfMayBeConditional"})
- public void finish() {
+ public void finish(boolean commit) {
boolean sync;
if (!F.isEmpty(dhtMap) || !F.isEmpty(nearMap))
- sync = finish(dhtMap, nearMap);
+ sync = finish(commit, dhtMap, nearMap);
else if (!commit && !F.isEmpty(tx.lockTransactionNodes()))
sync = rollbackLockTransactions(tx.lockTransactionNodes());
else
@@ -308,7 +292,6 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
* @return {@code True} in case there is at least one synchronous {@code MiniFuture} to wait for.
*/
private boolean rollbackLockTransactions(Collection<ClusterNode> nodes) {
- assert !commit;
assert !F.isEmpty(nodes);
if (tx.onePhaseCommit())
@@ -337,7 +320,7 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
tx.commitVersion(),
tx.threadId(),
tx.isolation(),
- commit,
+ false,
tx.isInvalidate(),
tx.system(),
tx.ioPolicy(),
@@ -390,11 +373,14 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
}
/**
+ * @param commit Commit flag.
* @param dhtMap DHT map.
* @param nearMap Near map.
* @return {@code True} in case there is at least one synchronous {@code MiniFuture} to wait for.
*/
- private boolean finish(Map<UUID, GridDistributedTxMapping> dhtMap, Map<UUID, GridDistributedTxMapping> nearMap) {
+ private boolean finish(boolean commit,
+ Map<UUID, GridDistributedTxMapping> dhtMap,
+ Map<UUID, GridDistributedTxMapping> nearMap) {
if (tx.onePhaseCommit())
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index b659abb..4e39e9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -503,52 +503,57 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
}
/**
+ * @param commit Commit flag.
* @param prepFut Prepare future.
* @param fut Finish future.
*/
- private void finishCommit(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinishFuture fut) {
+ private void finishTx(boolean commit, @Nullable IgniteInternalFuture prepFut, GridDhtTxFinishFuture fut) {
+ assert prepFut == null || prepFut.isDone();
+
boolean primarySync = syncMode() == PRIMARY_SYNC;
IgniteCheckedException err = null;
- try {
- if (prepFut != null)
- prepFut.get(); // Check for errors.
-
- if (finish(true)) {
- if (primarySync)
- sendFinishReply(true, null);
-
- fut.finish();
+ if (!commit && prepFut != null) {
+ try {
+ prepFut.get();
}
- else {
- err = new IgniteCheckedException("Failed to commit transaction: " + CU.txString(this));
-
- fut.onError(err);
+ catch (IgniteCheckedException e) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to prepare transaction [tx=" + this + ", e=" + e + ']');
+ }
+ finally {
+ prepFut = null;
}
}
- catch (IgniteTxOptimisticCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to optimistically prepare transaction [tx=" + this + ", e=" + e + ']');
- err = e;
+ try {
+ if (prepFut != null)
+ prepFut.get(); // Check for errors.
- fut.onError(e);
+ boolean finished = finish(commit);
+
+ if (!finished)
+ err = new IgniteCheckedException("Failed to finish transaction [commit=" + commit +
+ ", tx=" + CU.txString(this) + ']');
}
catch (IgniteCheckedException e) {
- U.error(log, "Failed to prepare transaction: " + this, e);
+ U.error(log, "Failed to finish transaction [commit=" + commit + ", tx=" + this + ']', e);
err = e;
-
- fut.onError(e);
}
- if (primarySync && err != null)
- sendFinishReply(true, err);
+ if (primarySync)
+ sendFinishReply(err);
+
+ if (err != null)
+ fut.rollbackOnError(err);
+ else
+ fut.finish(commit);
}
/** {@inheritDoc} */
- @SuppressWarnings({"ThrowableInstanceNeverThrown"})
+ @SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<IgniteInternalTx> commitAsync() {
if (log.isDebugEnabled())
log.debug("Committing dht local tx: " + this);
@@ -557,7 +562,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
if (pessimistic())
prepareAsync();
- final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/true);
+ final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true);
cctx.mvcc().addFuture(fut, fut.futureId());
@@ -565,11 +570,11 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
if (prep != null) {
if (prep.isDone())
- finishCommit(prep, fut);
+ finishTx(true, prep, fut);
else {
prep.listen(new CI1<IgniteInternalFuture<?>>() {
@Override public void apply(IgniteInternalFuture<?> f) {
- finishCommit(f, fut);
+ finishTx(true, f, fut);
}
});
}
@@ -577,7 +582,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
else {
assert optimistic();
- finishCommit(null, fut);
+ finishTx(true, null, fut);
}
return fut;
@@ -590,70 +595,26 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
PREP_FUT_UPD.compareAndSet(this, fut, null);
}
- /**
- * @param prepFut Prepare future.
- * @param fut Finish future.
- */
- private void finishRollback(@Nullable IgniteInternalFuture prepFut, GridDhtTxFinishFuture fut) {
- try {
- if (prepFut != null)
- prepFut.get();
- }
- catch (IgniteCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed to prepare or rollback transaction [tx=" + this + ", e=" + e + ']');
- }
-
- boolean primarySync = syncMode() == PRIMARY_SYNC;
-
- IgniteCheckedException err = null;
-
- try {
- if (finish(false) || state() == UNKNOWN) {
- if (primarySync)
- sendFinishReply(false, null);
-
- fut.finish();
- }
- else {
- err = new IgniteCheckedException("Failed to rollback transaction: " +
- CU.txString(GridDhtTxLocal.this));
-
- fut.onError(err);
- }
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to gracefully rollback transaction: " + CU.txString(GridDhtTxLocal.this),
- e);
-
- err = e;
-
- fut.onError(e);
- }
-
- if (primarySync && err != null)
- sendFinishReply(false, err);
- }
-
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<IgniteInternalTx> rollbackAsync() {
- GridDhtTxPrepareFuture prepFut = this.prepFut;
-
- final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*rollback*/false);
+ final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false);
cctx.mvcc().addFuture(fut, fut.futureId());
+ GridDhtTxPrepareFuture prepFut = this.prepFut;
+
if (prepFut != null) {
prepFut.complete();
prepFut.listen(new CI1<IgniteInternalFuture<?>>() {
@Override public void apply(IgniteInternalFuture<?> f) {
- finishRollback(f, fut);
+ finishTx(false, f, fut);
}
});
}
else
- finishRollback(null, fut);
+ finishTx(false, null, fut);
return fut;
}
@@ -672,7 +633,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
}
/** {@inheritDoc} */
- @Override protected void sendFinishReply(boolean commit, @Nullable Throwable err) {
+ @Override protected void sendFinishReply(@Nullable Throwable err) {
if (nearFinFutId != null) {
if (nearNodeId.equals(cctx.localNodeId())) {
if (log.isDebugEnabled())
@@ -701,8 +662,8 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
}
}
catch (Throwable ex) {
- U.error(log, "Failed to send finish response to node (transaction was " +
- (commit ? "committed" : "rolledback") + ") [txId=" + nearXidVersion() +
+ U.error(log, "Failed to send finish response to node [txId=" + nearXidVersion() +
+ ", txState=" + state() +
", dhtTxId=" + xidVersion() +
", node=" + nearNodeId +
", res=" + res + ']', ex);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 35dfb62..1d88d84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -236,10 +236,9 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
AffinityTopologyVersion topVer);
/**
- * @param commit Commit flag.
* @param err Error, if any.
*/
- protected abstract void sendFinishReply(boolean commit, @Nullable Throwable err);
+ protected abstract void sendFinishReply(@Nullable Throwable err);
/** {@inheritDoc} */
@Override public boolean needsCompletedVersions() {
@@ -249,7 +248,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
/**
* @return Versions for all pending locks that were in queue before tx locks were released.
*/
- public Collection<GridCacheVersion> pendingVersions() {
+ Collection<GridCacheVersion> pendingVersions() {
return pendingVers == null ? Collections.<GridCacheVersion>emptyList() : pendingVers;
}
@@ -726,7 +725,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
/*read*/read,
accessTtl,
filter == null ? CU.empty0() : filter,
- /**computeInvoke*/false);
+ /*computeInvoke*/false);
return ret;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 54bd543..9acab56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -298,34 +298,16 @@ public final class GridNearTxFinishFuture<K, V> extends GridCompoundIdentityFutu
if (isDone())
return false;
- if (err != null) {
- tx.commitError(err);
-
- boolean marked = tx.setRollbackOnly();
-
- if (err instanceof IgniteTxRollbackCheckedException) {
- if (marked) {
- try {
- tx.rollback();
- }
- catch (IgniteCheckedException ex) {
- U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
- }
- }
- }
- else if (tx.implicit() && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
- try {
- tx.close();
- }
- catch (IgniteCheckedException ex) {
- U.error(log, "Failed to invalidate transaction: " + tx, ex);
- }
- }
+ if (err != null)
+ tx.setRollbackOnly();
+
+ if (commit) {
+ if (tx.commitError() != null)
+ err = tx.commitError();
+ else if (err != null)
+ tx.commitError(err);
}
- if (commit && tx.commitError() != null)
- err = tx.commitError();
-
if (initialized() || err != null) {
if (tx.needCheckBackup()) {
assert tx.onePhaseCommit();
@@ -349,7 +331,15 @@ public final class GridNearTxFinishFuture<K, V> extends GridCompoundIdentityFutu
finishOnePhase(commit);
- tx.tmFinish(commit);
+ try {
+ tx.tmFinish(commit);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to finish tx: " + tx, e);
+
+ if (err == null)
+ err = e;
+ }
}
if (super.onDone(tx0, err)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index ed37059..0730300 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -231,7 +231,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter {
}
/** {@inheritDoc} */
- @Override protected void sendFinishReply(boolean commit, @Nullable Throwable err) {
+ @Override protected void sendFinishReply(@Nullable Throwable err) {
// We are in near transaction, do not send finish reply to local node.
}
@@ -1062,50 +1062,48 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter {
return new GridFinishedFuture<IgniteInternalTx>(this);
}
- final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/true);
+ final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, true);
cctx.mvcc().addFuture(fut, fut.futureId());
if (prep == null || prep.isDone()) {
assert prep != null || optimistic();
+ IgniteCheckedException err = null;
+
try {
if (prep != null)
prep.get(); // Check for errors of a parent future.
-
- fut.finish();
- }
- catch (IgniteTxOptimisticCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed optimistically to prepare transaction [tx=" + this + ", e=" + e + ']');
-
- fut.onError(e);
}
catch (IgniteCheckedException e) {
- U.error(log, "Failed to prepare transaction: " + this, e);
+ err = e;
- fut.onError(e);
+ U.error(log, "Failed to prepare transaction: " + this, e);
}
+
+ if (err != null)
+ fut.rollbackOnError(err);
+ else
+ fut.finish(true);
}
else
prep.listen(new CI1<IgniteInternalFuture<?>>() {
@Override public void apply(IgniteInternalFuture<?> f) {
+ IgniteCheckedException err = null;
+
try {
f.get(); // Check for errors of a parent future.
-
- fut.finish();
- }
- catch (IgniteTxOptimisticCheckedException e) {
- if (log.isDebugEnabled())
- log.debug("Failed optimistically to prepare transaction [tx=" + this + ", e=" + e + ']');
-
- fut.onError(e);
}
catch (IgniteCheckedException e) {
- U.error(log, "Failed to prepare transaction: " + this, e);
+ err = e;
- fut.onError(e);
+ U.error(log, "Failed to prepare transaction: " + this, e);
}
+
+ if (err != null)
+ fut.rollbackOnError(err);
+ else
+ fut.finish(true);
}
});
@@ -1121,7 +1119,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter {
if (log.isDebugEnabled())
log.debug("Rolling back colocated tx locally: " + this);
- final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/false);
+ final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, false);
cctx.mvcc().addFuture(fut, fut.futureId());
@@ -1138,7 +1136,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter {
e.getMessage() + ']');
}
- fut.finish();
+ fut.finish(false);
}
else
prep.listen(new CI1<IgniteInternalFuture<?>>() {
@@ -1151,7 +1149,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter {
e.getMessage() + ']');
}
- fut.finish();
+ fut.finish(false);
}
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 4b99079..b6e3c48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -472,13 +472,8 @@ public class IgniteTxHandler {
req.last());
if (tx.isRollbackOnly() && !tx.commitOnPrepare()) {
- try {
- if (tx.state() != TransactionState.ROLLED_BACK && tx.state() != TransactionState.ROLLING_BACK)
- tx.rollback();
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to rollback transaction: " + tx, e);
- }
+ if (tx.state() != TransactionState.ROLLED_BACK && tx.state() != TransactionState.ROLLING_BACK)
+ tx.rollbackAsync();
}
final GridDhtTxLocal tx0 = tx;
@@ -872,7 +867,7 @@ public class IgniteTxHandler {
U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
- IgniteInternalFuture<IgniteInternalTx> res = null;
+ IgniteInternalFuture<IgniteInternalTx> res;
IgniteInternalFuture<IgniteInternalTx> rollbackFut = tx.rollbackAsync();
@@ -884,7 +879,7 @@ public class IgniteTxHandler {
if (e instanceof Error)
throw (Error)e;
- return res == null ? new GridFinishedFuture<IgniteInternalTx>(e) : res;
+ return res;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 393fb1a..8d0a2b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1000,8 +1000,9 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
* Commits transaction to transaction manager. Used for one-phase commit transactions only.
*
* @param commit If {@code true} commits transaction, otherwise rollbacks.
+ * @throws IgniteCheckedException If failed.
*/
- public void tmFinish(boolean commit) {
+ public void tmFinish(boolean commit) throws IgniteCheckedException {
assert onePhaseCommit();
if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index e2e9868..036fb0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -1191,8 +1191,9 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* Commits a transaction.
*
* @param tx Transaction to commit.
+ * @throws IgniteCheckedException If failed.
*/
- public void commitTx(IgniteInternalTx tx) {
+ public void commitTx(IgniteInternalTx tx) throws IgniteCheckedException {
assert tx != null;
assert tx.state() == COMMITTING : "Invalid transaction state for commit from tm [state=" + tx.state() +
", expected=COMMITTING, tx=" + tx + ']';
@@ -1210,12 +1211,12 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
Boolean committed = committed0 != null && !committed0.equals(Boolean.FALSE);
// 1. Make sure that committed version has been recorded.
- if (!((committed != null && committed) || tx.writeSet().isEmpty() || tx.isSystemInvalidate())) {
+ if (!(committed || tx.writeSet().isEmpty() || tx.isSystemInvalidate())) {
uncommitTx(tx);
tx.errorWhenCommitting();
- throw new IgniteException("Missing commit version (consider increasing " +
+ throw new IgniteCheckedException("Missing commit version (consider increasing " +
IGNITE_MAX_COMPLETED_TX_COUNT + " system property) [ver=" + tx.xidVersion() +
", tx=" + tx.getClass().getSimpleName() + ']');
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
index 19e49f3..ac56d18 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheMissingCommitVersionSelfTest.java
@@ -43,7 +43,7 @@ public class GridCacheMissingCommitVersionSelfTest extends GridCommonAbstractTes
private volatile boolean putFailed;
/** */
- private String maxCompletedTxCount;
+ private String maxCompletedTxCnt;
/**
*/
@@ -53,7 +53,7 @@ public class GridCacheMissingCommitVersionSelfTest extends GridCommonAbstractTes
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration() throws Exception {
- maxCompletedTxCount = System.getProperty(IGNITE_MAX_COMPLETED_TX_COUNT);
+ maxCompletedTxCnt = System.getProperty(IGNITE_MAX_COMPLETED_TX_COUNT);
System.setProperty(IGNITE_MAX_COMPLETED_TX_COUNT, String.valueOf(5));
@@ -78,7 +78,7 @@ public class GridCacheMissingCommitVersionSelfTest extends GridCommonAbstractTes
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- System.setProperty(IGNITE_MAX_COMPLETED_TX_COUNT, maxCompletedTxCount != null ? maxCompletedTxCount : "");
+ System.setProperty(IGNITE_MAX_COMPLETED_TX_COUNT, maxCompletedTxCnt != null ? maxCompletedTxCnt : "");
super.afterTest();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1fde05d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
index b65b441..795ab81 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxStoreExceptionAbstractSelfTest.java
@@ -357,6 +357,7 @@ public abstract class IgniteTxStoreExceptionAbstractSelfTest extends GridCacheAb
/**
* @param key Key.
+ * @param putBefore If {@code true} expects non-null values.
* @throws Exception If failed.
*/
private void checkValue(final Integer key, boolean putBefore) throws Exception {
[11/50] [abbrv] ignite git commit: IGNITE-4231 - Hangs on compute
result serialization error. Fix
Posted by sb...@apache.org.
IGNITE-4231 - Hangs on compute result serialization error. Fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6d744db2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6d744db2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6d744db2
Branch: refs/heads/master
Commit: 6d744db27111c68e13b06ec99428a4c4148d97b6
Parents: ceb60d2
Author: dkarachentsev <dk...@gridgain.com>
Authored: Mon Dec 12 11:44:57 2016 +0300
Committer: dkarachentsev <dk...@gridgain.com>
Committed: Mon Dec 12 11:44:57 2016 +0300
----------------------------------------------------------------------
.../internal/processors/job/GridJobWorker.java | 69 +++++++-
.../closure/GridClosureSerializationTest.java | 177 +++++++++++++++++++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
3 files changed, 241 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6d744db2/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 5f38b29..f5c6a27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.job;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -790,6 +791,64 @@ public class GridJobWorker extends GridWorker implements GridTimeoutObject {
}
else {
try {
+ byte[] resBytes = null;
+ byte[] exBytes = null;
+ byte[] attrBytes = null;
+
+ boolean loc = ctx.localNodeId().equals(sndNode.id()) && !ctx.config().isMarshalLocalJobs();
+
+ Map<Object, Object> attrs = jobCtx.getAttributes();
+
+ // Try serialize response, and if exception - return to client.
+ if (!loc) {
+ try {
+ resBytes = U.marshal(marsh, res);
+ }
+ catch (IgniteCheckedException e) {
+ resBytes = U.marshal(marsh, null);
+
+ if (ex != null)
+ ex.addSuppressed(e);
+ else
+ ex = U.convertException(e);
+
+ U.error(log, "Failed to serialize job response [nodeId=" + taskNode.id() +
+ ", ses=" + ses + ", jobId=" + ses.getJobId() + ", job=" + job +
+ ", resCls=" + (res == null ? null : res.getClass()) + ']', e);
+ }
+
+ try {
+ attrBytes = U.marshal(marsh, attrs);
+ }
+ catch (IgniteCheckedException e) {
+ attrBytes = U.marshal(marsh, Collections.emptyMap());
+
+ if (ex != null)
+ ex.addSuppressed(e);
+ else
+ ex = U.convertException(e);
+
+ U.error(log, "Failed to serialize job attributes [nodeId=" + taskNode.id() +
+ ", ses=" + ses + ", jobId=" + ses.getJobId() + ", job=" + job +
+ ", attrs=" + attrs + ']', e);
+ }
+
+ try {
+ exBytes = U.marshal(marsh, ex);
+ }
+ catch (IgniteCheckedException e) {
+ String msg = "Failed to serialize job exception [nodeId=" + taskNode.id() +
+ ", ses=" + ses + ", jobId=" + ses.getJobId() + ", job=" + job +
+ ", msg=\"" + e.getMessage() + "\"]";
+
+ ex = new IgniteException(msg);
+
+ U.error(log, msg, e);
+
+ exBytes = U.marshal(marsh, ex);
+ }
+ }
+
if (ex != null) {
if (isStarted) {
// Job failed.
@@ -804,19 +863,15 @@ public class GridJobWorker extends GridWorker implements GridTimeoutObject {
else if (!internal && ctx.event().isRecordable(EVT_JOB_FINISHED))
evts = addEvent(evts, EVT_JOB_FINISHED, /*no message for success. */null);
- boolean loc = ctx.localNodeId().equals(sndNode.id()) && !ctx.config().isMarshalLocalJobs();
-
- Map<Object, Object> attrs = jobCtx.getAttributes();
-
GridJobExecuteResponse jobRes = new GridJobExecuteResponse(
ctx.localNodeId(),
ses.getId(),
ses.getJobId(),
- loc ? null : U.marshal(marsh, ex),
+ exBytes,
loc ? ex : null,
- loc ? null: U.marshal(marsh, res),
+ resBytes,
loc ? res : null,
- loc ? null : U.marshal(marsh, attrs),
+ attrBytes,
loc ? attrs : null,
isCancelled(),
retry ? ctx.cache().context().exchange().readyAffinityVersion() : null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6d744db2/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
new file mode 100644
index 0000000..2426dd7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.closure;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.JobContextResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests handling of job result serialization error.
+ */
+public class GridClosureSerializationTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setMarshaller(null);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrid(0);
+ startGrid(1);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+ public void testSerializationFailure() throws Exception {
+ final IgniteEx ignite0 = grid(0);
+ final IgniteEx ignite1 = grid(1);
+
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
+ @Override public Object call() throws Exception {
+ return new CaseClass.CaseClass2();
+ }
+ });
+
+ return null;
+ }
+ }, BinaryObjectException.class, null);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+ public void testExceptionSerializationFailure() throws Exception {
+ final IgniteEx ignite0 = grid(0);
+ final IgniteEx ignite1 = grid(1);
+
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
+ @Override public Object call() throws Exception {
+ throw new BrokenException();
+ }
+ });
+
+ return null;
+ }
+ }, IgniteException.class, null);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "Convert2Lambda"})
+ public void testAttributesSerializationFailure() throws Exception {
+ final IgniteEx ignite0 = grid(0);
+ final IgniteEx ignite1 = grid(1);
+
+ GridTestUtils.assertThrows(null, new Callable<Object>() {
+ @JobContextResource
+ private ComputeJobContext jobCtx;
+
+ @Override public Object call() throws Exception {
+ ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
+ @Override public Object call() throws Exception {
+ jobCtx.setAttribute("test-attr", new BrokenAttribute());
+
+ return null;
+ }
+ });
+
+ return null;
+ }
+ }, IgniteException.class, null);
+ }
+
+ /**
+ * Binary marshaller will fail because subclass defines other field with different case.
+ */
+ @SuppressWarnings("unused")
+ private static class CaseClass {
+ /** */
+ private String val;
+
+ /**
+ *
+ */
+ private static class CaseClass2 extends CaseClass {
+ /** */
+ private String vAl;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class BrokenAttribute implements Externalizable {
+ /** {@inheritDoc} */
+ @Override public void writeExternal(final ObjectOutput out) throws IOException {
+ throw new IOException("Test exception");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+ throw new IOException("Test exception");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class BrokenException extends Exception implements Externalizable {
+ /** {@inheritDoc} */
+ @Override public void writeExternal(final ObjectOutput out) throws IOException {
+ throw new IOException("Test exception");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException {
+ throw new IOException("Test exception");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6d744db2/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 6ab0885..1c1fcf7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.cache.IgniteDaemonNodeMarshallerCac
import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheConcurrentReadWriteTest;
import org.apache.ignite.internal.processors.cache.OffHeapTieredTransactionSelfTest;
import org.apache.ignite.internal.processors.closure.GridClosureProcessorSelfTest;
+import org.apache.ignite.internal.processors.closure.GridClosureSerializationTest;
import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
import org.apache.ignite.internal.processors.continuous.GridMessageListenSelfTest;
import org.apache.ignite.internal.processors.odbc.OdbcProcessorValidationSelfTest;
@@ -109,6 +110,7 @@ public class IgniteBasicTestSuite extends TestSuite {
suite.addTestSuite(GridProductVersionSelfTest.class);
suite.addTestSuite(GridAffinityProcessorRendezvousSelfTest.class);
suite.addTestSuite(GridClosureProcessorSelfTest.class);
+ suite.addTestSuite(GridClosureSerializationTest.class);
suite.addTestSuite(ClosureServiceClientsNodesTest.class);
suite.addTestSuite(GridStartStopSelfTest.class);
suite.addTestSuite(GridProjectionForCachesSelfTest.class);
[06/50] [abbrv] ignite git commit: Fixed tests for IGNITE-4379 commit.
Posted by sb...@apache.org.
Fixed tests for IGNITE-4379 commit.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6fd8bf63
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6fd8bf63
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6fd8bf63
Branch: refs/heads/master
Commit: 6fd8bf6338470275e687a686044c7d935d3714ff
Parents: 6e48563
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Wed Dec 7 18:49:06 2016 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Fri Dec 9 15:56:21 2016 +0300
----------------------------------------------------------------------
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 19 ---------------
...niteCachePartitionedFieldsQuerySelfTest.java | 25 ++++++++++++++++++++
2 files changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6fd8bf63/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
index 2c35501..d5f02eb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
@@ -28,12 +28,10 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
-import javax.cache.Cache;
import javax.cache.CacheException;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.affinity.AffinityKey;
@@ -652,23 +650,6 @@ public abstract class IgniteCacheAbstractFieldsQuerySelfTest extends GridCommonA
}
/** @throws Exception If failed. */
- public void testLocalQuery() throws Exception {
- IgniteCache<Object, Object> cache = grid(0).cache( null);
-
- int expected = 0;
-
- for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
- if(e.getValue() instanceof Integer)
- expected++;
- }
-
- QueryCursor<List<?>> qry = cache
- .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
-
- assertEquals(expected, qry.getAll().size());
- }
-
- /** @throws Exception If failed. */
public void testQueryIntegersWithJoin() throws Exception {
QueryCursor<List<?>> qry = grid(0).cache(null).query(new SqlFieldsQuery(
"select i._KEY, i._VAL, j._KEY, j._VAL from Integer i join Integer j where i._VAL >= 100"));
http://git-wip-us.apache.org/repos/asf/ignite/blob/6fd8bf63/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
index 653947e..af5845d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java
@@ -17,7 +17,13 @@
package org.apache.ignite.internal.processors.cache.distributed.near;
+import java.util.List;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractFieldsQuerySelfTest;
@@ -54,4 +60,23 @@ public class IgniteCachePartitionedFieldsQuerySelfTest extends IgniteCacheAbstra
return cc;
}
+
+ /** @throws Exception If failed. */
+ public void testLocalQuery() throws Exception {
+ IgniteCache<Object, Object> cache = grid(0).cache( null);
+
+ awaitPartitionMapExchange(true, true, null);
+
+ int expected = 0;
+
+ for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
+ if(e.getValue() instanceof Integer)
+ expected++;
+ }
+
+ QueryCursor<List<?>> qry = cache
+ .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
+
+ assertEquals(expected, qry.getAll().size());
+ }
}
\ No newline at end of file
[40/50] [abbrv] ignite git commit: ignite-2412 Do not call 'asyncOp'
for synchronous operations
Posted by sb...@apache.org.
ignite-2412 Do not call 'asyncOp' for synchronous operations
(cherry picked from commit c530d47)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ad785cbd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ad785cbd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ad785cbd
Branch: refs/heads/master
Commit: ad785cbd192ca3f34d62bc2155f61a74f4962102
Parents: 28dab6e
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 16 19:23:29 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 19 11:40:18 2016 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheAdapter.java | 721 +++++++------------
.../processors/cache/IgniteCacheProxy.java | 8 -
.../dht/atomic/GridDhtAtomicCache.java | 472 +++++++-----
.../dht/colocated/GridDhtColocatedCache.java | 13 -
.../distributed/near/GridNearAtomicCache.java | 65 +-
.../local/atomic/GridLocalAtomicCache.java | 177 +----
...nabledMultiNodeLongTxTimeoutFullApiTest.java | 2 +-
...lockMessageSystemPoolStarvationSelfTest.java | 14 +-
8 files changed, 597 insertions(+), 875 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index a8d9f1d..5f0b8a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -248,16 +248,16 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** Grid configuration. */
@GridToStringExclude
- protected IgniteConfiguration gridCfg;
+ private IgniteConfiguration gridCfg;
/** Cache metrics. */
protected CacheMetricsImpl metrics;
/** Cache localMxBean. */
- protected CacheMetricsMXBean localMxBean;
+ private CacheMetricsMXBean locMxBean;
/** Cache mxBean. */
- protected CacheMetricsMXBean clusterMxBean;
+ private CacheMetricsMXBean clusterMxBean;
/** Logger. */
protected IgniteLogger log;
@@ -288,9 +288,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** Asynchronous operations limit semaphore. */
private Semaphore asyncOpsSem;
- /** */
- protected volatile boolean asyncToggled;
-
/** {@inheritDoc} */
@Override public String name() {
return cacheCfg.getName();
@@ -334,7 +331,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
metrics = new CacheMetricsImpl(ctx);
- localMxBean = new CacheLocalMetricsMXBeanImpl(this);
+ locMxBean = new CacheLocalMetricsMXBeanImpl(this);
clusterMxBean = new CacheClusterMetricsMXBeanImpl(this);
FileSystemConfiguration[] igfsCfgs = gridCfg.getFileSystemConfiguration();
@@ -367,18 +364,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/**
- * Toggles async flag if someone calls {@code withAsync()}
- * on proxy and since that we have to properly handle all cache
- * operations (sync and async) to put them in proper sequence.
- *
- * TODO: https://issues.apache.org/jira/browse/IGNITE-4393
- */
- void toggleAsync() {
- if (!asyncToggled)
- asyncToggled = true;
- }
-
- /**
* Prints memory stats.
*/
public void printMemoryStats() {
@@ -471,49 +456,49 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
public abstract GridCachePreloader preloader();
/** {@inheritDoc} */
- @Override public Affinity<K> affinity() {
+ @Override public final Affinity<K> affinity() {
return aff;
}
/** {@inheritDoc} */
@SuppressWarnings({"unchecked", "RedundantCast"})
- @Override public <K1, V1> IgniteInternalCache<K1, V1> cache() {
+ @Override public final <K1, V1> IgniteInternalCache<K1, V1> cache() {
return (IgniteInternalCache<K1, V1>)this;
}
/** {@inheritDoc} */
- @Override public GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
+ @Override public final GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
CacheOperationContext opCtx = new CacheOperationContext(false, subjId, false, null, false, null);
return new GridCacheProxyImpl<>(ctx, this, opCtx);
}
/** {@inheritDoc} */
- @Override public boolean skipStore() {
+ @Override public final boolean skipStore() {
return false;
}
/** {@inheritDoc} */
- @Override public GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
+ @Override public final GridCacheProxyImpl<K, V> setSkipStore(boolean skipStore) {
CacheOperationContext opCtx = new CacheOperationContext(true, null, false, null, false, null);
return new GridCacheProxyImpl<>(ctx, this, opCtx);
}
/** {@inheritDoc} */
- @Override public <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
+ @Override public final <K1, V1> GridCacheProxyImpl<K1, V1> keepBinary() {
CacheOperationContext opCtx = new CacheOperationContext(false, null, true, null, false, null);
return new GridCacheProxyImpl<>((GridCacheContext<K1, V1>)ctx, (GridCacheAdapter<K1, V1>)this, opCtx);
}
/** {@inheritDoc} */
- @Nullable @Override public ExpiryPolicy expiry() {
+ @Nullable @Override public final ExpiryPolicy expiry() {
return null;
}
/** {@inheritDoc} */
- @Override public GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
+ @Override public final GridCacheProxyImpl<K, V> withExpiryPolicy(ExpiryPolicy plc) {
assert !CU.isUtilityCache(ctx.name());
assert !CU.isAtomicsCache(ctx.name());
assert !CU.isMarshallerCache(ctx.name());
@@ -524,14 +509,14 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public IgniteInternalCache<K, V> withNoRetries() {
+ @Override public final IgniteInternalCache<K, V> withNoRetries() {
CacheOperationContext opCtx = new CacheOperationContext(false, null, false, null, true, null);
return new GridCacheProxyImpl<>(ctx, this, opCtx);
}
/** {@inheritDoc} */
- @Override public CacheConfiguration configuration() {
+ @Override public final CacheConfiguration configuration() {
return ctx.config();
}
@@ -630,7 +615,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public boolean isEmpty() {
+ @Override public final boolean isEmpty() {
try {
return localSize(CachePeekModes.ONHEAP_ONLY) == 0;
}
@@ -640,7 +625,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public boolean containsKey(K key) {
+ @Override public final boolean containsKey(K key) {
try {
return containsKeyAsync(key).get();
}
@@ -667,7 +652,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public boolean containsKeys(Collection<? extends K> keys) {
+ @Override public final boolean containsKeys(Collection<? extends K> keys) {
try {
return containsKeysAsync(keys).get();
}
@@ -677,7 +662,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
+ @Override public final IgniteInternalFuture<Boolean> containsKeysAsync(final Collection<? extends K> keys) {
A.notNull(keys, "keys");
return getAllAsync(
@@ -708,7 +693,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
+ @Override public final Iterable<Cache.Entry<K, V>> localEntries(CachePeekMode[] peekModes) throws IgniteCheckedException {
assert peekModes != null;
ctx.checkSecurity(SecurityPermission.CACHE_READ);
@@ -765,7 +750,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** {@inheritDoc} */
@SuppressWarnings("ForLoopReplaceableByForEach")
- @Nullable @Override public V localPeek(K key,
+ @Nullable @Override public final V localPeek(K key,
CachePeekMode[] peekModes,
@Nullable IgniteCacheExpiryPolicy plc)
throws IgniteCheckedException {
@@ -907,7 +892,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
*
* @param ldr Class loader to undeploy.
*/
- public void onUndeploy(ClassLoader ldr) {
+ public final void onUndeploy(ClassLoader ldr) {
ctx.deploy().onUndeploy(ldr, context());
}
@@ -916,7 +901,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param key Entry key.
* @return Entry or <tt>null</tt>.
*/
- @Nullable public GridCacheEntryEx peekEx(KeyCacheObject key) {
+ @Nullable public final GridCacheEntryEx peekEx(KeyCacheObject key) {
return entry0(key, ctx.affinity().affinityTopologyVersion(), false, false);
}
@@ -925,7 +910,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param key Entry key.
* @return Entry or <tt>null</tt>.
*/
- @Nullable public GridCacheEntryEx peekEx(Object key) {
+ @Nullable public final GridCacheEntryEx peekEx(Object key) {
return entry0(ctx.toCacheKeyObject(key), ctx.affinity().affinityTopologyVersion(), false, false);
}
@@ -933,7 +918,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param key Entry key.
* @return Entry (never {@code null}).
*/
- public GridCacheEntryEx entryEx(Object key) {
+ public final GridCacheEntryEx entryEx(Object key) {
return entryEx(ctx.toCacheKeyObject(key), false);
}
@@ -941,7 +926,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param key Entry key.
* @return Entry (never {@code null}).
*/
- public GridCacheEntryEx entryEx(KeyCacheObject key) {
+ public final GridCacheEntryEx entryEx(KeyCacheObject key) {
return entryEx(key, false);
}
@@ -996,24 +981,24 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/**
* @return Set of internal cached entry representations.
*/
- public Iterable<? extends GridCacheEntryEx> entries() {
+ public final Iterable<? extends GridCacheEntryEx> entries() {
return allEntries();
}
/**
* @return Set of internal cached entry representations.
*/
- public Iterable<? extends GridCacheEntryEx> allEntries() {
+ public final Iterable<? extends GridCacheEntryEx> allEntries() {
return map.entries();
}
/** {@inheritDoc} */
- @Override public Set<Cache.Entry<K, V>> entrySet() {
+ @Override public final Set<Cache.Entry<K, V>> entrySet() {
return entrySet((CacheEntryPredicate[])null);
}
/** {@inheritDoc} */
- @Override public Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
+ @Override public final Set<Cache.Entry<K, V>> entrySetx(final CacheEntryPredicate... filter) {
boolean keepBinary = ctx.keepBinary();
return new EntrySet(map.entrySet(filter), keepBinary);
@@ -1025,17 +1010,17 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public Set<K> keySet() {
+ @Override public final Set<K> keySet() {
return new KeySet(map.entrySet());
}
/** {@inheritDoc} */
- @Override public Set<K> keySetx() {
+ @Override public final Set<K> keySetx() {
return keySet();
}
/** {@inheritDoc} */
- @Override public Set<K> primaryKeySet() {
+ @Override public final Set<K> primaryKeySet() {
return new KeySet(map.entrySet(CU.cachePrimary(ctx.grid().affinity(ctx.name()), ctx.localNode())));
}
@@ -1057,7 +1042,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param filter Filters.
* @return Collection of cached values.
*/
- public Iterable<V> values(final CacheEntryPredicate... filter) {
+ public final Iterable<V> values(final CacheEntryPredicate... filter) {
return new Iterable<V>() {
@Override public Iterator<V> iterator() {
return new Iterator<V>() {
@@ -1083,12 +1068,12 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
*
* @param key Entry key.
*/
- public void removeIfObsolete(KeyCacheObject key) {
+ public final void removeIfObsolete(KeyCacheObject key) {
assert key != null;
GridCacheMapEntry entry = map.getEntry(key);
- if (entry.obsolete())
+ if (entry != null && entry.obsolete())
removeEntry(entry);
}
@@ -1272,11 +1257,11 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/**
* @param entry Removes entry from cache if currently mapped value is the same as passed.
*/
- public void removeEntry(GridCacheEntryEx entry) {
- boolean removed = map.removeEntry(entry);
+ public final void removeEntry(GridCacheEntryEx entry) {
+ boolean rmvd = map.removeEntry(entry);
if (log.isDebugEnabled()) {
- if (removed)
+ if (rmvd)
log.debug("Removed entry from cache: " + entry);
else
log.debug("Remove will not be done for key (entry got replaced or removed): " + entry.key());
@@ -1311,7 +1296,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public V getForcePrimary(K key) throws IgniteCheckedException {
+ @Override public final V getForcePrimary(K key) throws IgniteCheckedException {
String taskName = ctx.kernalContext().job().currentTaskName();
return getAllAsync(
@@ -1328,7 +1313,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
+ @Override public final IgniteInternalFuture<V> getForcePrimaryAsync(final K key) {
String taskName = ctx.kernalContext().job().currentTaskName();
return getAllAsync(
@@ -1349,7 +1334,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- public V getTopologySafe(K key) throws IgniteCheckedException {
+ public final V getTopologySafe(K key) throws IgniteCheckedException {
String taskName = ctx.kernalContext().job().currentTaskName();
return getAllAsync(
@@ -1366,12 +1351,12 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Nullable @Override public Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
+ @Nullable @Override public final Map<K, V> getAllOutTx(Set<? extends K> keys) throws IgniteCheckedException {
return getAllOutTxAsync(keys).get();
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
+ @Override public final IgniteInternalFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
String taskName = ctx.kernalContext().job().currentTaskName();
return getAllAsync(keys,
@@ -1385,15 +1370,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
false);
}
- /**
- * @param key Key.
- * @param topVer Topology version.
- * @return Entry.
- */
- @Nullable protected GridCacheEntryEx entryExSafe(KeyCacheObject key, AffinityTopologyVersion topVer) {
- return entryEx(key);
- }
-
/** {@inheritDoc} */
@Nullable @Override public V get(K key) throws IgniteCheckedException {
A.notNull(key, "key");
@@ -1533,14 +1509,14 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
+ @Override public final Map<K, V> getAll(@Nullable Collection<? extends K> keys) throws IgniteCheckedException {
A.notNull(keys, "keys");
boolean statsEnabled = ctx.config().isStatisticsEnabled();
long start = statsEnabled ? System.nanoTime() : 0L;
- Map<K, V> map = getAll(keys, !ctx.keepBinary(), false);
+ Map<K, V> map = getAll0(keys, !ctx.keepBinary(), false);
if (ctx.config().getInterceptor() != null)
map = interceptGet(keys, map);
@@ -1560,7 +1536,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
long start = statsEnabled ? System.nanoTime() : 0L;
- Map<K, T2<V, GridCacheVersion>> map = (Map<K, T2<V, GridCacheVersion>>)getAll(keys, !ctx.keepBinary(), true);
+ Map<K, T2<V, GridCacheVersion>> map = (Map<K, T2<V, GridCacheVersion>>)getAll0(keys, !ctx.keepBinary(), true);
Collection<CacheEntry<K, V>> res = new HashSet<>();
@@ -1875,7 +1851,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param needVer If {@code true} returns values as tuples containing value and version.
* @return Future.
*/
- public final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
+ protected final <K1, V1> IgniteInternalFuture<Map<K1, V1>> getAllAsync0(
@Nullable final Collection<KeyCacheObject> keys,
final boolean readThrough,
boolean checkTx,
@@ -2138,7 +2114,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public V getAndPut(K key, V val) throws IgniteCheckedException {
+ @Override public final V getAndPut(K key, V val) throws IgniteCheckedException {
return getAndPut(key, val, null);
}
@@ -2160,7 +2136,24 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
- V prevVal = syncOp(new SyncOp<V>(true) {
+ V prevVal = getAndPut0(key, val, filter);
+
+ if (statsEnabled)
+ metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
+
+ return prevVal;
+ }
+
+ /**
+ * @param key Key.
+ * @param val Value.
+ * @param filter Optional filter.
+ * @return Previous value.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected V getAndPut0(final K key, final V val, @Nullable final CacheEntryPredicate filter)
+ throws IgniteCheckedException {
+ return syncOp(new SyncOp<V>(true) {
@Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
return (V)tx.putAsync(ctx, null, key, val, true, filter).get().value();
}
@@ -2169,15 +2162,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "put [key=" + key + ", val=" + val + ", filter=" + filter + ']';
}
});
-
- if (statsEnabled)
- metrics0().addPutAndGetTimeNanos(System.nanoTime() - start);
-
- return prevVal;
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
+ @Override public final IgniteInternalFuture<V> getAndPutAsync(K key, V val) {
return getAndPutAsync(key, val, null);
}
@@ -2187,11 +2175,16 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param filter Filter.
* @return Put operation future.
*/
- public IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+ protected final IgniteInternalFuture<V> getAndPutAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
final boolean statsEnabled = ctx.config().isStatisticsEnabled();
final long start = statsEnabled ? System.nanoTime() : 0L;
+ A.notNull(key, "key", val, "val");
+
+ if (keyCheck)
+ validateCacheKey(key);
+
IgniteInternalFuture<V> fut = getAndPutAsync0(key, val, filter);
if (statsEnabled)
@@ -2206,13 +2199,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param filter Optional filter.
* @return Put operation future.
*/
- public IgniteInternalFuture<V> getAndPutAsync0(final K key, final V val,
- @Nullable final CacheEntryPredicate filter) {
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
+ public IgniteInternalFuture<V> getAndPutAsync0(final K key,
+ final V val,
+ @Nullable final CacheEntryPredicate filter)
+ {
return asyncOp(new AsyncOp<V>() {
@Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
return tx.putAsync(ctx, readyTopVer, key, val, true, filter)
@@ -2226,7 +2216,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public boolean put(final K key, final V val) throws IgniteCheckedException {
+ @Override public final boolean put(final K key, final V val) throws IgniteCheckedException {
return put(key, val, null);
}
@@ -2250,7 +2240,26 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
- Boolean stored = syncOp(new SyncOp<Boolean>(true) {
+ boolean stored = put0(key, val, filter);
+
+ if (statsEnabled && stored)
+ metrics0().addPutTimeNanos(System.nanoTime() - start);
+
+ return stored;
+ }
+
+ /**
+ * @param key Key.
+ * @param val Value.
+ * @param filter Filter.
+ * @return {@code True} if optional filter passed and value was stored in cache,
+ * {@code false} otherwise. Note that this method will return {@code true} if filter is not
+ * specified.
+ * @throws IgniteCheckedException If put operation failed.
+ */
+ protected boolean put0(final K key, final V val, final CacheEntryPredicate filter)
+ throws IgniteCheckedException {
+ Boolean res = syncOp(new SyncOp<Boolean>(true) {
@Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
return tx.putAsync(ctx, null, key, val, false, filter).get().success();
}
@@ -2260,10 +2269,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
});
- if (statsEnabled)
- metrics0().addPutTimeNanos(System.nanoTime() - start);
+ assert res != null;
- return stored;
+ return res;
}
/** {@inheritDoc} */
@@ -2305,7 +2313,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Nullable @Override public <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
+ @Nullable @Override public final <T> EntryProcessorResult<T> invoke(@Nullable AffinityTopologyVersion topVer,
K key,
EntryProcessor<K, V, T> entryProcessor,
Object... args) throws IgniteCheckedException {
@@ -2538,7 +2546,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> putAsync(K key, V val) {
+ @Override public final IgniteInternalFuture<Boolean> putAsync(K key, V val) {
return putAsync(key, val, null);
}
@@ -2548,9 +2556,12 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param filter Filter.
* @return Put future.
*/
- public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+ public final IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
A.notNull(key, "key", val, "val");
+ if (keyCheck)
+ validateCacheKey(key);
+
final boolean statsEnabled = ctx.config().isStatisticsEnabled();
final long start = statsEnabled ? System.nanoTime() : 0L;
@@ -2571,9 +2582,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
*/
public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
@Nullable final CacheEntryPredicate filter) {
- if (keyCheck)
- validateCacheKey(key);
-
return asyncOp(new AsyncOp<Boolean>() {
@Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
return tx.putAsync(ctx,
@@ -2598,267 +2606,82 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Nullable @Override public V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- return syncOp(new SyncOp<V>(true) {
- @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- return (V)tx.putAsync(ctx, null, key, val, true, ctx.noVal()).get().value();
- }
-
- @Override public String toString() {
- return "putIfAbsent [key=" + key + ", val=" + val + ']';
- }
- });
+ @Nullable @Override public final V getAndPutIfAbsent(final K key, final V val) throws IgniteCheckedException {
+ return getAndPut(key, val, ctx.noVal());
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
- @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
- return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.noVal())
- .chain((IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
- }
-
- @Override public String toString() {
- return "putIfAbsentAsync [key=" + key + ", val=" + val + ']';
- }
- });
-
- if (statsEnabled)
- fut.listen(new UpdatePutTimeStatClosure<V>(metrics0(), start));
-
- return fut;
+ @Override public final IgniteInternalFuture<V> getAndPutIfAbsentAsync(final K key, final V val) {
+ return getAndPutAsync(key, val, ctx.noVal());
}
/** {@inheritDoc} */
- @Override public boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
- boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- Boolean stored = syncOp(new SyncOp<Boolean>(true) {
- @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- return tx.putAsync(ctx, null, key, val, false, ctx.noVal()).get().success();
- }
-
- @Override public String toString() {
- return "putxIfAbsent [key=" + key + ", val=" + val + ']';
- }
- });
-
- if (statsEnabled && stored)
- metrics0().addPutTimeNanos(System.nanoTime() - start);
-
- return stored;
+ @Override public final boolean putIfAbsent(final K key, final V val) throws IgniteCheckedException {
+ return put(key, val, ctx.noVal());
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
- @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
- return tx.putAsync(ctx,
- readyTopVer,
- key,
- val,
- false,
- ctx.noVal()).chain(
- (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
- }
-
- @Override public String toString() {
- return "putxIfAbsentAsync [key=" + key + ", val=" + val + ']';
- }
- });
-
- if (statsEnabled)
- fut.listen(new UpdatePutTimeStatClosure<Boolean>(metrics0(), start));
-
- return fut;
+ @Override public final IgniteInternalFuture<Boolean> putIfAbsentAsync(final K key, final V val) {
+ return putAsync(key, val, ctx.noVal());
}
/** {@inheritDoc} */
- @Nullable @Override public V getAndReplace(final K key, final V val) throws IgniteCheckedException {
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- return syncOp(new SyncOp<V>(true) {
- @Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- return (V)tx.putAsync(ctx, null, key, val, true, ctx.hasVal()).get().value();
- }
-
- @Override public String toString() {
- return "replace [key=" + key + ", val=" + val + ']';
- }
- });
+ @Nullable @Override public final V getAndReplace(final K key, final V val) throws IgniteCheckedException {
+ return getAndPut(key, val, ctx.hasVal());
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
- @Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
- return tx.putAsync(ctx, readyTopVer, key, val, true, ctx.hasVal()).chain(
- (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, V>)RET2VAL);
- }
-
- @Override public String toString() {
- return "replaceAsync [key=" + key + ", val=" + val + ']';
- }
- });
-
- if (statsEnabled)
- fut.listen(new UpdatePutAndGetTimeStatClosure<V>(metrics0(), start));
-
- return fut;
+ @Override public final IgniteInternalFuture<V> getAndReplaceAsync(final K key, final V val) {
+ return getAndPutAsync(key, val, ctx.hasVal());
}
/** {@inheritDoc} */
- @Override public boolean replace(final K key, final V val) throws IgniteCheckedException {
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- return syncOp(new SyncOp<Boolean>(true) {
- @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- return tx.putAsync(ctx, null, key, val, false, ctx.hasVal()).get().success();
- }
-
- @Override public String toString() {
- return "replacex [key=" + key + ", val=" + val + ']';
- }
- });
+ @Override public final boolean replace(final K key, final V val) throws IgniteCheckedException {
+ return put(key, val, ctx.hasVal());
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- return asyncOp(new AsyncOp<Boolean>() {
- @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
- return tx.putAsync(ctx, readyTopVer, key, val, false, ctx.hasVal()).chain(
- (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>) RET2FLAG);
- }
-
- @Override public String toString() {
- return "replacexAsync [key=" + key + ", val=" + val + ']';
- }
- });
+ @Override public final IgniteInternalFuture<Boolean> replaceAsync(final K key, final V val) {
+ return putAsync(key, val, ctx.hasVal());
}
/** {@inheritDoc} */
- @Override public boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
- A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
- if (keyCheck)
- validateCacheKey(key);
+ @Override public final boolean replace(final K key, final V oldVal, final V newVal) throws IgniteCheckedException {
+ A.notNull(oldVal, "oldVal");
- return syncOp(new SyncOp<Boolean>(true) {
- @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- // Register before hiding in the filter.
- if (ctx.deploymentEnabled())
- ctx.deploy().registerClass(oldVal);
-
- return tx.putAsync(ctx, null, key, newVal, false, ctx.equalsVal(oldVal)).get()
- .success();
- }
-
- @Override public String toString() {
- return "replace [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
- }
- });
+ return put(key, newVal, ctx.equalsVal(oldVal));
}
/** {@inheritDoc} */
@Override public IgniteInternalFuture<Boolean> replaceAsync(final K key, final V oldVal, final V newVal) {
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
+ A.notNull(oldVal, "oldVal");
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
- if (keyCheck)
- validateCacheKey(key);
-
- IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
- @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
- // Register before hiding in the filter.
- if (ctx.deploymentEnabled()) {
- try {
- ctx.deploy().registerClass(oldVal);
- }
- catch (IgniteCheckedException e) {
- return new GridFinishedFuture<>(e);
- }
- }
-
- return tx.putAsync(ctx, readyTopVer, key, newVal, false, ctx.equalsVal(oldVal)).chain(
- (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
- }
-
- @Override public String toString() {
- return "replaceAsync [key=" + key + ", oldVal=" + oldVal + ", newVal=" + newVal + ']';
- }
- });
-
- if (statsEnabled)
- fut.listen(new UpdatePutAndGetTimeStatClosure<Boolean>(metrics0(), start));
-
- return fut;
+ return putAsync(key, newVal, ctx.equalsVal(oldVal));
}
/** {@inheritDoc} */
@Override public void putAll(@Nullable final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+ if (F.isEmpty(m))
+ return;
+
boolean statsEnabled = ctx.config().isStatisticsEnabled();
long start = statsEnabled ? System.nanoTime() : 0L;
- if (F.isEmpty(m))
- return;
-
if (keyCheck)
validateCacheKeys(m.keySet());
+ putAll0(m);
+
+ if (statsEnabled)
+ metrics0().addPutTimeNanos(System.nanoTime() - start);
+ }
+
+ /**
+ * @param m Map.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected void putAll0(final Map<? extends K, ? extends V> m) throws IgniteCheckedException {
syncOp(new SyncInOp(m.size() == 1) {
@Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
tx.putAllAsync(ctx, null, m, false).get();
@@ -2868,9 +2691,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "putAll [map=" + m + ']';
}
});
-
- if (statsEnabled)
- metrics0().addPutTimeNanos(System.nanoTime() - start);
}
/** {@inheritDoc} */
@@ -2881,6 +2701,14 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(m.keySet());
+ return putAllAsync0(m);
+ }
+
+ /**
+ * @param m Map.
+ * @return Future.
+ */
+ protected IgniteInternalFuture<?> putAllAsync0(final Map<? extends K, ? extends V> m) {
return asyncOp(new AsyncOp(m.keySet()) {
@Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
return tx.putAllAsync(ctx,
@@ -2906,11 +2734,25 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
+ V prevVal = getAndRemove0(key);
+
+ if (statsEnabled)
+ metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
+
+ return prevVal;
+ }
+
+ /**
+ * @param key Key.
+ * @return Previous value.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected V getAndRemove0(final K key) throws IgniteCheckedException {
final boolean keepBinary = ctx.keepBinary();
- V prevVal = syncOp(new SyncOp<V>(true) {
+ return syncOp(new SyncOp<V>(true) {
@Override public V op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
+ K key0 = keepBinary ? (K) ctx.toCacheKeyObject(key) : key;
V ret = tx.removeAllAsync(ctx,
null,
@@ -2920,9 +2762,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/*singleRmv*/false).get().value();
if (ctx.config().getInterceptor() != null) {
- K key = keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
+ K key = keepBinary ? (K) ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
- return (V)ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
+ return (V) ctx.config().getInterceptor().onBeforeRemove(new CacheEntryImpl(key, ret)).get2();
}
return ret;
@@ -2932,11 +2774,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "remove [key=" + key + ']';
}
});
-
- if (statsEnabled)
- metrics0().addRemoveAndGetTimeNanos(System.nanoTime() - start);
-
- return prevVal;
}
/** {@inheritDoc} */
@@ -2950,7 +2787,20 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
- IgniteInternalFuture<V> fut = asyncOp(new AsyncOp<V>() {
+ IgniteInternalFuture<V> fut = getAndRemoveAsync0(key);
+
+ if (statsEnabled)
+ fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
+
+ return fut;
+ }
+
+ /**
+ * @param key Key.
+ * @return Future.
+ */
+ protected IgniteInternalFuture<V> getAndRemoveAsync0(final K key) {
+ return asyncOp(new AsyncOp<V>() {
@Override public IgniteInternalFuture<V> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
// TODO should we invoke interceptor here?
return tx.removeAllAsync(ctx,
@@ -2965,11 +2815,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "removeAsync [key=" + key + ']';
}
});
-
- if (statsEnabled)
- fut.listen(new UpdateRemoveTimeStatClosure<V>(metrics0(), start));
-
- return fut;
}
/** {@inheritDoc} */
@@ -3002,6 +2847,17 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(keys);
+ removeAll0(keys);
+
+ if (statsEnabled)
+ metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+ }
+
+ /**
+ * @param keys Keys.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected void removeAll0(final Collection<? extends K> keys) throws IgniteCheckedException {
syncOp(new SyncInOp(keys.size() == 1) {
@Override public void inOp(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
tx.removeAllAsync(ctx,
@@ -3016,24 +2872,34 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "removeAll [keys=" + keys + ']';
}
});
-
- if (statsEnabled)
- metrics0().addRemoveTimeNanos(System.nanoTime() - start);
}
/** {@inheritDoc} */
@Override public IgniteInternalFuture<?> removeAllAsync(@Nullable final Collection<? extends K> keys) {
+ if (F.isEmpty(keys))
+ return new GridFinishedFuture<Object>();
+
final boolean statsEnabled = ctx.config().isStatisticsEnabled();
final long start = statsEnabled ? System.nanoTime() : 0L;
- if (F.isEmpty(keys))
- return new GridFinishedFuture<Object>();
-
if (keyCheck)
validateCacheKeys(keys);
- IgniteInternalFuture<Object> fut = asyncOp(new AsyncOp(keys) {
+ IgniteInternalFuture<Object> fut = removeAllAsync0(keys);
+
+ if (statsEnabled)
+ fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+
+ return fut;
+ }
+
+ /**
+ * @param keys Keys.
+ * @return Future.
+ */
+ protected IgniteInternalFuture<Object> removeAllAsync0(final Collection<? extends K> keys) {
+ return asyncOp(new AsyncOp(keys) {
@Override public IgniteInternalFuture<?> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
return tx.removeAllAsync(ctx,
readyTopVer,
@@ -3047,15 +2913,20 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "removeAllAsync [keys=" + keys + ']';
}
});
-
- if (statsEnabled)
- fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
- return fut;
}
/** {@inheritDoc} */
@Override public boolean remove(final K key) throws IgniteCheckedException {
+ return remove(key, (CacheEntryPredicate)null);
+ }
+
+ /**
+ * @param key Key.
+ * @param filter Filter.
+ * @return {@code True} if entry was removed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean remove(final K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
boolean statsEnabled = ctx.config().isStatisticsEnabled();
long start = statsEnabled ? System.nanoTime() : 0L;
@@ -3065,13 +2936,27 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
- boolean rmv = syncOp(new SyncOp<Boolean>(true) {
+ boolean rmv = remove0(key, filter);
+
+ if (statsEnabled && rmv)
+ metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+
+ return rmv;
+ }
+
+ /**
+ * @param key Key.
+ * @return {@code True} if entry was removed.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected boolean remove0(final K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
+ Boolean res = syncOp(new SyncOp<Boolean>(true) {
@Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
return tx.removeAllAsync(ctx,
null,
Collections.singletonList(key),
/*retval*/false,
- null,
+ filter,
/*singleRmv*/true).get().success();
}
@@ -3080,10 +2965,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
});
- if (statsEnabled && rmv)
- metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+ assert res != null;
- return rmv;
+ return res;
}
/** {@inheritDoc} */
@@ -3108,7 +2992,21 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
- IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
+ IgniteInternalFuture<Boolean> fut = removeAsync0(key, filter);
+
+ if (statsEnabled)
+ fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
+
+ return fut;
+ }
+
+ /**
+ * @param key Key.
+ * @param filter Filter.
+ * @return Future.
+ */
+ protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable final CacheEntryPredicate filter) {
+ return asyncOp(new AsyncOp<Boolean>() {
@Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
return tx.removeAllAsync(ctx,
readyTopVer,
@@ -3123,11 +3021,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return "removeAsync [key=" + key + ", filter=" + filter + ']';
}
});
-
- if (statsEnabled)
- fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
- return fut;
}
/** {@inheritDoc} */
@@ -3169,86 +3062,21 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/** {@inheritDoc} */
- @Override public boolean remove(final K key, final V val) throws IgniteCheckedException {
- boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- boolean rmv = syncOp(new SyncOp<Boolean>(true) {
- @Override public Boolean op(IgniteTxLocalAdapter tx) throws IgniteCheckedException {
- // Register before hiding in the filter.
- if (ctx.deploymentEnabled())
- ctx.deploy().registerClass(val);
-
- return tx.removeAllAsync(ctx,
- null,
- Collections.singletonList(key),
- /*retval*/false,
- ctx.equalsVal(val),
- /*singleRmv*/false).get().success();
- }
-
- @Override public String toString() {
- return "remove [key=" + key + ", val=" + val + ']';
- }
- });
+ @Override public final boolean remove(final K key, final V val) throws IgniteCheckedException {
+ A.notNull(val, "val");
- if (statsEnabled && rmv)
- metrics0().addRemoveTimeNanos(System.nanoTime() - start);
-
- return rmv;
+ return remove(key, ctx.equalsVal(val));
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key", val, "val");
-
- if (keyCheck)
- validateCacheKey(key);
-
- IgniteInternalFuture<Boolean> fut = asyncOp(new AsyncOp<Boolean>() {
- @Override public IgniteInternalFuture<Boolean> op(IgniteTxLocalAdapter tx, AffinityTopologyVersion readyTopVer) {
- // Register before hiding in the filter.
- if (ctx.deploymentEnabled()) {
- try {
- ctx.deploy().registerClass(val);
- }
- catch (IgniteCheckedException e) {
- return new GridFinishedFuture<>(e);
- }
- }
-
- return tx.removeAllAsync(ctx,
- readyTopVer,
- Collections.singletonList(key),
- /*retval*/false,
- ctx.equalsVal(val),
- /*singleRmv*/false).chain(
- (IgniteClosure<IgniteInternalFuture<GridCacheReturn>, Boolean>)RET2FLAG);
- }
+ @Override public final IgniteInternalFuture<Boolean> removeAsync(final K key, final V val) {
+ A.notNull(key, "val");
- @Override public String toString() {
- return "removeAsync [key=" + key + ", val=" + val + ']';
- }
- });
-
- if (statsEnabled)
- fut.listen(new UpdateRemoveTimeStatClosure<Boolean>(metrics0(), start));
-
- return fut;
+ return removeAsync(key, ctx.equalsVal(val));
}
/** {@inheritDoc} */
- @Override public CacheMetrics clusterMetrics() {
+ @Override public final CacheMetrics clusterMetrics() {
return clusterMetrics(ctx.grid().cluster().forCacheNodes(ctx.name()));
}
@@ -3277,7 +3105,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** {@inheritDoc} */
@Override public CacheMetricsMXBean localMxBean() {
- return localMxBean;
+ return locMxBean;
}
/** {@inheritDoc} */
@@ -4607,9 +4435,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @return Failed future if waiting was interrupted.
*/
@Nullable protected <T> IgniteInternalFuture<T> asyncOpAcquire() {
- if (!asyncToggled)
- return null;
-
try {
if (asyncOpsSem != null)
asyncOpsSem.acquire();
@@ -4627,8 +4452,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/**
* Releases asynchronous operations permit, if limited.
*/
- protected void asyncOpRelease() {
- if (asyncOpsSem != null && asyncToggled)
+ private void asyncOpRelease() {
+ if (asyncOpsSem != null)
asyncOpsSem.release();
}
@@ -4793,12 +4618,10 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @return Cached value.
* @throws IgniteCheckedException If failed.
*/
- @Nullable public V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
- checkJta();
-
+ @Nullable public final V get(K key, boolean deserializeBinary, final boolean needVer) throws IgniteCheckedException {
String taskName = ctx.kernalContext().job().currentTaskName();
- return get(key, taskName, deserializeBinary, needVer);
+ return get0(key, taskName, deserializeBinary, needVer);
}
/**
@@ -4809,11 +4632,13 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @return Cached value.
* @throws IgniteCheckedException If failed.
*/
- protected V get(
+ protected V get0(
final K key,
String taskName,
boolean deserializeBinary,
boolean needVer) throws IgniteCheckedException {
+ checkJta();
+
try {
return getAsync(key,
!ctx.config().isReadFromBackup(),
@@ -4867,7 +4692,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @return Map of cached values.
* @throws IgniteCheckedException If read failed.
*/
- public Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary,
+ protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary,
boolean needVer) throws IgniteCheckedException {
checkJta();
@@ -4922,7 +4747,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param key Cache key.
* @throws IllegalArgumentException If validation fails.
*/
- protected void validateCacheKey(Object key) {
+ protected final void validateCacheKey(Object key) {
if (keyCheck) {
CU.validateCacheKey(key);
@@ -4937,7 +4762,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param keys Cache keys.
* @throws IgniteException If validation fails.
*/
- protected void validateCacheKeys(Iterable<?> keys) {
+ protected final void validateCacheKeys(Iterable<?> keys) {
if (keys == null)
return;
@@ -4958,7 +4783,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param deserializeBinary Deserialize binary flag.
* @return Public API iterator.
*/
- protected Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
+ protected final Iterator<Cache.Entry<K, V>> iterator(final Iterator<? extends GridCacheEntryEx> it,
final boolean deserializeBinary) {
return new Iterator<Cache.Entry<K, V>>() {
{
@@ -5276,7 +5101,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param plc Explicitly specified expiry policy for cache operation.
* @return Expiry policy wrapper.
*/
- @Nullable public IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
+ @Nullable public final IgniteCacheExpiryPolicy expiryPolicy(@Nullable ExpiryPolicy plc) {
if (plc == null)
plc = ctx.expiry();
@@ -5401,7 +5226,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @param opCtx Operation context.
* @return Operation future.
*/
- protected IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
+ private IgniteInternalFuture<T> waitTopologyFuture(IgniteInternalFuture<?> topFut,
final AffinityTopologyVersion topVer,
final IgniteTxLocalAdapter tx,
final CacheOperationContext opCtx) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index b9e6e82..f87fa1d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -334,14 +334,6 @@ public class IgniteCacheProxy<K, V> extends AsyncSupportAdapter<IgniteCache<K, V
}
/** {@inheritDoc} */
- @Override public IgniteCache<K, V> withAsync() {
- if (delegate instanceof GridCacheAdapter)
- ((GridCacheAdapter)delegate).toggleAsync();
-
- return super.withAsync();
- }
-
- /** {@inheritDoc} */
@Override public IgniteCache<K, V> withSkipStore() {
return skipStore();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 0e60ff4..a67a903 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -139,7 +139,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
private CI2<GridNearAtomicAbstractUpdateRequest, GridNearAtomicUpdateResponse> updateReplyClos;
/** Pending */
- private GridDeferredAckMessageSender deferredUpdateMessageSender;
+ private GridDeferredAckMessageSender deferredUpdateMsgSnd;
/** */
private GridNearAtomicCache<K, V> near;
@@ -174,6 +174,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
/** {@inheritDoc} */
+ @Override protected void checkJta() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public boolean isDhtAtomic() {
return true;
}
@@ -235,7 +240,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Override public void start() throws IgniteCheckedException {
super.start();
- deferredUpdateMessageSender = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
+ deferredUpdateMsgSnd = new GridDeferredAckMessageSender(ctx.time(), ctx.closures()) {
@Override public int getTimeout() {
return DEFERRED_UPDATE_RESPONSE_TIMEOUT;
}
@@ -447,7 +452,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
/** {@inheritDoc} */
@Override public void stop() {
- deferredUpdateMessageSender.stop();
+ deferredUpdateMsgSnd.stop();
}
/**
@@ -463,7 +468,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
/** {@inheritDoc} */
- @Override protected V get(K key, String taskName, boolean deserializeBinary, boolean needVer)
+ @Override protected V get0(K key, String taskName, boolean deserializeBinary, boolean needVer)
throws IgniteCheckedException {
ctx.checkSecurity(SecurityPermission.CACHE_READ);
@@ -540,6 +545,21 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
/** {@inheritDoc} */
+ @Override protected Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+ throws IgniteCheckedException {
+ return getAllAsyncInternal(keys,
+ !ctx.config().isReadFromBackup(),
+ true,
+ null,
+ ctx.kernalContext().job().currentTaskName(),
+ deserializeBinary,
+ false,
+ true,
+ needVer,
+ false).get();
+ }
+
+ /** {@inheritDoc} */
@Override public IgniteInternalFuture<Map<K, V>> getAllAsync(
@Nullable final Collection<? extends K> keys,
final boolean forcePrimary,
@@ -551,6 +571,43 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
final boolean canRemap,
final boolean needVer
) {
+ return getAllAsyncInternal(keys,
+ forcePrimary,
+ skipTx,
+ subjId,
+ taskName,
+ deserializeBinary,
+ skipVals,
+ canRemap,
+ needVer,
+ true);
+ }
+
+ /**
+ * @param keys Keys.
+ * @param forcePrimary Force primary flag.
+ * @param skipTx Skip tx flag.
+ * @param subjId Subject ID.
+ * @param taskName Task name.
+ * @param deserializeBinary Deserialize binary flag.
+ * @param skipVals Skip values flag.
+ * @param canRemap Can remap flag.
+ * @param needVer Need version flag.
+ * @param asyncOp Async operation flag.
+ * @return Future.
+ */
+ private IgniteInternalFuture<Map<K, V>> getAllAsyncInternal(
+ @Nullable final Collection<? extends K> keys,
+ final boolean forcePrimary,
+ boolean skipTx,
+ @Nullable UUID subjId,
+ final String taskName,
+ final boolean deserializeBinary,
+ final boolean skipVals,
+ final boolean canRemap,
+ final boolean needVer,
+ boolean asyncOp
+ ) {
ctx.checkSecurity(SecurityPermission.CACHE_READ);
if (F.isEmpty(keys))
@@ -561,7 +618,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
CacheOperationContext opCtx = ctx.operationContextPerCall();
- subjId = ctx.subjectIdPerCall(null, opCtx);
+ subjId = ctx.subjectIdPerCall(subjId, opCtx);
final UUID subjId0 = subjId;
@@ -569,57 +626,91 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
final boolean skipStore = opCtx != null && opCtx.skipStore();
- return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
- @Override public IgniteInternalFuture<Map<K, V>> apply() {
- return getAllAsync0(ctx.cacheKeysView(keys),
- forcePrimary,
- subjId0,
- taskName,
- deserializeBinary,
- expiryPlc,
- skipVals,
- skipStore,
- canRemap,
- needVer);
- }
- });
+ if (asyncOp) {
+ return asyncOp(new CO<IgniteInternalFuture<Map<K, V>>>() {
+ @Override public IgniteInternalFuture<Map<K, V>> apply() {
+ return getAllAsync0(ctx.cacheKeysView(keys),
+ forcePrimary,
+ subjId0,
+ taskName,
+ deserializeBinary,
+ expiryPlc,
+ skipVals,
+ skipStore,
+ canRemap,
+ needVer);
+ }
+ });
+ }
+ else {
+ return getAllAsync0(ctx.cacheKeysView(keys),
+ forcePrimary,
+ subjId0,
+ taskName,
+ deserializeBinary,
+ expiryPlc,
+ skipVals,
+ skipStore,
+ canRemap,
+ needVer);
+ }
}
/** {@inheritDoc} */
- @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
- return getAndPutAsync0(key, val, filter).get();
+ @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+ return (V)update0(
+ key,
+ val,
+ null,
+ null,
+ true,
+ filter,
+ true,
+ false).get();
}
/** {@inheritDoc} */
- @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
- return putAsync(key, val, filter).get();
+ @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
+ Boolean res = (Boolean)update0(
+ key,
+ val,
+ null,
+ null,
+ false,
+ filter,
+ true,
+ false).get();
+
+ assert res != null;
+
+ return res;
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
- A.notNull(key, "key", val, "val");
-
- return updateAsync0(
+ return update0(
key,
val,
null,
null,
true,
filter,
+ true,
true);
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
- return updateAsync0(
+ return update0(
key,
val,
null,
null,
false,
filter,
+ true,
true);
}
@@ -627,84 +718,34 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
A.notNull(key, "key", val, "val");
- return (V)updateAsync0(
+ return (V) update0(
key,
val,
null,
null,
true,
null,
+ false,
false).get();
}
/** {@inheritDoc} */
- @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
- return getAndPutIfAbsentAsync(key, val).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
- A.notNull(key, "key", val, "val");
-
- return getAndPutAsync(key, val, ctx.noVal());
- }
-
- /** {@inheritDoc} */
- @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
- return putIfAbsentAsync(key, val).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
- A.notNull(key, "key", val, "val");
-
- return putAsync(key, val, ctx.noVal());
- }
-
- /** {@inheritDoc} */
- @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
- return getAndReplaceAsync(key, val).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
- A.notNull(key, "key", val, "val");
-
- return getAndPutAsync(key, val, ctx.hasVal());
- }
-
- /** {@inheritDoc} */
- @Override public boolean replace(K key, V val) throws IgniteCheckedException {
- return replaceAsync(key, val).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
- A.notNull(key, "key", val, "val");
-
- return putAsync(key, val, ctx.hasVal());
- }
-
- /** {@inheritDoc} */
- @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
- return replaceAsync(key, oldVal, newVal).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
- A.notNull(key, "key", oldVal, "oldVal", newVal, "newVal");
-
- return putAsync(key, newVal, ctx.equalsVal(oldVal));
- }
-
- /** {@inheritDoc} */
- @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
- putAllAsync(m).get();
+ @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
+ updateAll0(m,
+ null,
+ null,
+ null,
+ null,
+ false,
+ false,
+ true,
+ UPDATE,
+ false).get();
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
- return updateAllAsync0(m,
+ @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
+ return updateAll0(m,
null,
null,
null,
@@ -712,7 +753,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
false,
true,
- UPDATE).chain(RET2NULL);
+ UPDATE,
+ true).chain(RET2NULL);
}
/** {@inheritDoc} */
@@ -725,7 +767,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Override public IgniteInternalFuture<?> putAllConflictAsync(Map<KeyCacheObject, GridCacheDrInfo> conflictMap) {
ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
- return updateAllAsync0(null,
+ return updateAll0(null,
null,
null,
conflictMap,
@@ -733,57 +775,40 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
false,
true,
- UPDATE);
+ UPDATE,
+ true);
}
/** {@inheritDoc} */
- @Override public V getAndRemove(K key) throws IgniteCheckedException {
- return getAndRemoveAsync(key).get();
+ @Override public V getAndRemove0(K key) throws IgniteCheckedException {
+ return (V)remove0(key, true, null, false).get();
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
- A.notNull(key, "key");
-
- return removeAsync0(key, true, null);
+ @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
+ return remove0(key, true, null, true);
}
/** {@inheritDoc} */
- @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
- removeAllAsync(keys).get();
+ @Override protected void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
+ removeAllAsync0(keys, null, false, false, false).get();
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
- A.notNull(keys, "keys");
-
- return removeAllAsync0(keys, null, false, false).chain(RET2NULL);
+ @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
+ return removeAllAsync0(keys, null, false, false, true).chain(RET2NULL);
}
/** {@inheritDoc} */
- @Override public boolean remove(K key) throws IgniteCheckedException {
- return removeAsync(key, (CacheEntryPredicate)null).get();
+ @Override protected boolean remove0(K key, CacheEntryPredicate filter) throws IgniteCheckedException {
+ return (Boolean)remove0(key, false, filter, false).get();
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
- A.notNull(key, "key");
-
- return removeAsync0(key, false, filter);
- }
-
- /** {@inheritDoc} */
- @Override public boolean remove(K key, V val) throws IgniteCheckedException {
- return removeAsync(key, val).get();
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
- A.notNull(key, "key", val, "val");
-
- return removeAsync(key, ctx.equalsVal(val));
+ @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
+ return remove0(key, false, filter, true);
}
/** {@inheritDoc} */
@@ -796,7 +821,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Override public IgniteInternalFuture<?> removeAllConflictAsync(Map<KeyCacheObject, GridCacheVersion> conflictMap) {
ctx.dr().onReceiveCacheEntriesReceived(conflictMap.size());
- return removeAllAsync0(null, conflictMap, false, false);
+ return removeAllAsync0(null, conflictMap, false, false, true);
}
/**
@@ -811,10 +836,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @return Future.
*/
@SuppressWarnings("unchecked")
- protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
- if (!asyncToggled)
- return op.apply();
-
+ private <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
IgniteInternalFuture<T> fail = asyncOpAcquire();
if (fail != null)
@@ -871,7 +893,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
/** {@inheritDoc} */
@Override public <T> EntryProcessorResult<T> invoke(K key, EntryProcessor<K, V, T> entryProcessor, Object... args)
throws IgniteCheckedException {
- IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invokeAsync(key, entryProcessor, args);
+ IgniteInternalFuture<EntryProcessorResult<T>> invokeFut = invoke0(false, key, entryProcessor, args);
EntryProcessorResult<T> res = invokeFut.get();
@@ -881,16 +903,30 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
/** {@inheritDoc} */
@Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys,
EntryProcessor<K, V, T> entryProcessor,
- Object... args)
- throws IgniteCheckedException {
- return invokeAllAsync(keys, entryProcessor, args).get();
+ Object... args) throws IgniteCheckedException
+ {
+ return invokeAll0(false, keys, entryProcessor, args).get();
}
/** {@inheritDoc} */
- @SuppressWarnings("unchecked")
@Override public <T> IgniteInternalFuture<EntryProcessorResult<T>> invokeAsync(K key,
EntryProcessor<K, V, T> entryProcessor,
Object... args) {
+ return invoke0(true, key, entryProcessor, args);
+ }
+
+ /**
+ * @param async Async operation flag.
+ * @param key Key.
+ * @param entryProcessor Entry processor.
+ * @param args Entry processor arguments.
+ * @return Future.
+ */
+ private <T> IgniteInternalFuture<EntryProcessorResult<T>> invoke0(
+ boolean async,
+ K key,
+ EntryProcessor<K, V, T> entryProcessor,
+ Object... args) {
A.notNull(key, "key", entryProcessor, "entryProcessor");
if (keyCheck)
@@ -900,14 +936,15 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
- IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = updateAsync0(
+ IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut = update0(
key,
null,
entryProcessor,
args,
false,
null,
- true);
+ true,
+ async);
return fut.chain(new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, EntryProcessorResult<T>>() {
@Override public EntryProcessorResult<T> applyx(IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> fut)
@@ -940,6 +977,21 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Override public <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys,
final EntryProcessor<K, V, T> entryProcessor,
Object... args) {
+ return invokeAll0(true, keys, entryProcessor, args);
+ }
+
+ /**
+ * @param async Async operation flag.
+ * @param keys Keys.
+ * @param entryProcessor Entry processor.
+ * @param args Entry processor arguments.
+ * @return Future.
+ */
+ private <T> IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> invokeAll0(
+ boolean async,
+ Set<? extends K> keys,
+ final EntryProcessor<K, V, T> entryProcessor,
+ Object... args) {
A.notNull(keys, "keys", entryProcessor, "entryProcessor");
if (keyCheck)
@@ -955,7 +1007,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
- IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAllAsync0(null,
+ IgniteInternalFuture<Map<K, EntryProcessorResult<T>>> resFut = updateAll0(null,
invokeMap,
args,
null,
@@ -963,7 +1015,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
false,
true,
- TRANSFORM);
+ TRANSFORM,
+ async);
return resFut.chain(
new CX1<IgniteInternalFuture<Map<K, EntryProcessorResult<T>>>, Map<K, EntryProcessorResult<T>>>() {
@@ -981,7 +1034,21 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(
Map<? extends K, ? extends EntryProcessor<K, V, T>> map,
Object... args) throws IgniteCheckedException {
- return invokeAllAsync(map, args).get();
+ A.notNull(map, "map");
+
+ if (keyCheck)
+ validateCacheKeys(map.keySet());
+
+ return (Map<K, EntryProcessorResult<T>>)updateAll0(null,
+ map,
+ args,
+ null,
+ null,
+ false,
+ false,
+ true,
+ TRANSFORM,
+ false).get();
}
/** {@inheritDoc} */
@@ -994,7 +1061,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
if (keyCheck)
validateCacheKeys(map.keySet());
- return updateAllAsync0(null,
+ return updateAll0(null,
map,
args,
null,
@@ -1002,7 +1069,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
false,
true,
- TRANSFORM);
+ TRANSFORM,
+ true);
}
/**
@@ -1016,10 +1084,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param retval Return value required flag.
* @param rawRetval Return {@code GridCacheReturn} instance.
* @param waitTopFut Whether to wait for topology future.
+ * @param async Async operation flag.
* @return Completion future.
*/
@SuppressWarnings("ConstantConditions")
- private IgniteInternalFuture updateAllAsync0(
+ private IgniteInternalFuture updateAll0(
@Nullable Map<? extends K, ? extends V> map,
@Nullable Map<? extends K, ? extends EntryProcessor> invokeMap,
@Nullable Object[] invokeArgs,
@@ -1028,7 +1097,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
final boolean retval,
final boolean rawRetval,
final boolean waitTopFut,
- final GridCacheOperation op
+ final GridCacheOperation op,
+ boolean async
) {
assert ctx.updatesAllowed();
@@ -1105,13 +1175,20 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
waitTopFut);
- return asyncOp(new CO<IgniteInternalFuture<Object>>() {
- @Override public IgniteInternalFuture<Object> apply() {
- updateFut.map();
+ if (async) {
+ return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+ @Override public IgniteInternalFuture<Object> apply() {
+ updateFut.map();
- return updateFut;
- }
- });
+ return updateFut;
+ }
+ });
+ }
+ else {
+ updateFut.map();
+
+ return updateFut;
+ }
}
/**
@@ -1124,16 +1201,18 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param retval Return value flag.
* @param filter Filter.
* @param waitTopFut Whether to wait for topology future.
+ * @param async Async operation flag.
* @return Future.
*/
- private IgniteInternalFuture updateAsync0(
+ private IgniteInternalFuture update0(
K key,
@Nullable V val,
@Nullable EntryProcessor proc,
@Nullable Object[] invokeArgs,
final boolean retval,
@Nullable final CacheEntryPredicate filter,
- final boolean waitTopFut
+ final boolean waitTopFut,
+ boolean async
) {
assert val == null || proc == null;
@@ -1146,13 +1225,20 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
final GridNearAtomicAbstractUpdateFuture updateFut =
createSingleUpdateFuture(key, val, proc, invokeArgs, retval, filter, waitTopFut);
- return asyncOp(new CO<IgniteInternalFuture<Object>>() {
- @Override public IgniteInternalFuture<Object> apply() {
- updateFut.map();
+ if (async) {
+ return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+ @Override public IgniteInternalFuture<Object> apply() {
+ updateFut.map();
- return updateFut;
- }
- });
+ return updateFut;
+ }
+ });
+ }
+ else {
+ updateFut.map();
+
+ return updateFut;
+ }
}
/**
@@ -1161,33 +1247,38 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param key Key.
* @param retval Whether to return
* @param filter Filter.
+ * @param async Async operation flag.
* @return Future.
*/
- private IgniteInternalFuture removeAsync0(K key, final boolean retval,
- @Nullable CacheEntryPredicate filter) {
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
+ private IgniteInternalFuture remove0(K key, final boolean retval,
+ @Nullable CacheEntryPredicate filter,
+ boolean async) {
assert ctx.updatesAllowed();
- validateCacheKey(key);
-
ctx.checkSecurity(SecurityPermission.CACHE_REMOVE);
- final GridNearAtomicAbstractUpdateFuture updateFut =
- createSingleUpdateFuture(key, null, null, null, retval, filter, true);
+ final GridNearAtomicAbstractUpdateFuture updateFut = createSingleUpdateFuture(key,
+ null,
+ null,
+ null,
+ retval,
+ filter,
+ true);
- if (statsEnabled)
- updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+ if (async) {
+ return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+ @Override public IgniteInternalFuture<Object> apply() {
+ updateFut.map();
- return asyncOp(new CO<IgniteInternalFuture<Object>>() {
- @Override public IgniteInternalFuture<Object> apply() {
- updateFut.map();
+ return updateFut;
+ }
+ });
+ }
+ else {
+ updateFut.map();
- return updateFut;
- }
- });
+ return updateFut;
+ }
}
/**
@@ -1326,14 +1417,11 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
@Nullable Collection<? extends K> keys,
@Nullable Map<KeyCacheObject, GridCacheVersion> conflictMap,
final boolean retval,
- boolean rawRetval
+ boolean rawRetval,
+ boolean async
) {
assert ctx.updatesAllowed();
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
assert keys != null || conflictMap != null;
if (keyCheck)
@@ -1380,16 +1468,20 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
opCtx != null && opCtx.noRetries() ? 1 : MAX_RETRIES,
true);
- if (statsEnabled)
- updateFut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
+ if (async) {
+ return asyncOp(new CO<IgniteInternalFuture<Object>>() {
+ @Override public IgniteInternalFuture<Object> apply() {
+ updateFut.map();
- return asyncOp(new CO<IgniteInternalFuture<Object>>() {
- @Override public IgniteInternalFuture<Object> apply() {
- updateFut.map();
+ return updateFut;
+ }
+ });
+ }
+ else {
+ updateFut.map();
- return updateFut;
- }
- });
+ return updateFut;
+ }
}
/**
@@ -3248,7 +3340,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param ver Version to ack.
*/
private void sendDeferredUpdateResponse(UUID nodeId, GridCacheVersion ver) {
- deferredUpdateMessageSender.sendDeferredAckMessage(nodeId, ver);
+ deferredUpdateMsgSnd.sendDeferredAckMessage(nodeId, ver);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 176a90f..9cf400d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -327,19 +327,6 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
needVer);
}
- /** {@inheritDoc} */
- @Override protected GridCacheEntryEx entryExSafe(
- KeyCacheObject key,
- AffinityTopologyVersion topVer
- ) {
- try {
- return ctx.affinity().localNode(key, topVer) ? entryEx(key) : null;
- }
- catch (GridDhtInvalidPartitionException ignored) {
- return null;
- }
- }
-
/**
* @param keys Keys to load.
* @param readThrough Read through flag.
[24/50] [abbrv] ignite git commit: IGNITE-4270: Hadoop: implemented
striped mapper output. This closes #1334.
Posted by sb...@apache.org.
IGNITE-4270: Hadoop: implemented striped mapper output. This closes #1334.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a976c425
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a976c425
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a976c425
Branch: refs/heads/master
Commit: a976c42590f786ff977999736be5436e7dac9d87
Parents: 58c3380
Author: devozerov <vo...@gridgain.com>
Authored: Fri Dec 9 12:01:40 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:19 2016 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 6 +
.../processors/hadoop/HadoopJobProperty.java | 7 +
.../hadoop/HadoopMapperAwareTaskOutput.java | 32 ++
.../processors/hadoop/HadoopTaskInfo.java | 43 ++
.../shuffle/HadoopDirectShuffleMessage.java | 243 ++++++++++++
.../processors/hadoop/HadoopMapperUtils.java | 56 +++
.../hadoop/impl/v2/HadoopV2Context.java | 11 +
.../hadoop/impl/v2/HadoopV2MapTask.java | 10 +
.../hadoop/jobtracker/HadoopJobTracker.java | 4 +
.../hadoop/shuffle/HadoopShuffle.java | 23 +-
.../hadoop/shuffle/HadoopShuffleJob.java | 389 ++++++++++++++-----
.../shuffle/HadoopShuffleRemoteState.java | 5 +-
.../shuffle/direct/HadoopDirectDataInput.java | 166 ++++++++
.../shuffle/direct/HadoopDirectDataOutput.java | 221 +++++++++++
.../direct/HadoopDirectDataOutputContext.java | 100 +++++
.../direct/HadoopDirectDataOutputState.java | 54 +++
.../child/HadoopChildProcessRunner.java | 2 +-
.../impl/HadoopMapReduceEmbeddedSelfTest.java | 22 +-
18 files changed, 1287 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 4ffb220..504e683 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -122,6 +122,7 @@ import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck;
import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishRequest;
import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleFinishResponse;
import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopDirectShuffleMessage;
import org.apache.ignite.internal.processors.igfs.IgfsAckMessage;
import org.apache.ignite.internal.processors.igfs.IgfsBlockKey;
import org.apache.ignite.internal.processors.igfs.IgfsBlocksMessage;
@@ -170,6 +171,11 @@ public class GridIoMessageFactory implements MessageFactory {
Message msg = null;
switch (type) {
+ case -42:
+ msg = new HadoopDirectShuffleMessage();
+
+ break;
+
case -41:
msg = new HadoopShuffleFinishResponse();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index e713caa..1f0ef1b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -91,6 +91,13 @@ public enum HadoopJobProperty {
SHUFFLE_MSG_SIZE("ignite.shuffle.message.size"),
/**
+ * Whether to stripe mapper output for remote reducers.
+ * <p>
+ * Defaults to {@code false}.
+ */
+ SHUFFLE_MAPPER_STRIPED_OUTPUT("ignite.shuffle.mapper.striped.output"),
+
+ /**
* Shuffle job throttle in milliseconds. When job is executed with separate shuffle thread, this parameter
* controls sleep duration between iterations through intermediate reducer maps.
* <p>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java
new file mode 100644
index 0000000..1d6637c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperAwareTaskOutput.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Special output type with callback invoked when mapper finished writing data.
+ */
+public interface HadoopMapperAwareTaskOutput extends HadoopTaskOutput {
+ /**
+ * Callback invoked when mapper finished writing data.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void onMapperFinished() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
index b76fb85..3509367 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskInfo.java
@@ -46,6 +46,12 @@ public class HadoopTaskInfo implements Externalizable {
/** */
private HadoopInputSplit inputSplit;
+ /** Whether mapper index is set. */
+ private boolean mapperIdxSet;
+
+ /** Current mapper index. */
+ private int mapperIdx;
+
/**
* For {@link Externalizable}.
*/
@@ -78,6 +84,13 @@ public class HadoopTaskInfo implements Externalizable {
out.writeInt(taskNum);
out.writeInt(attempt);
out.writeObject(inputSplit);
+
+ if (mapperIdxSet) {
+ out.writeBoolean(true);
+ out.writeInt(mapperIdx);
+ }
+ else
+ out.writeBoolean(false);
}
/** {@inheritDoc} */
@@ -87,6 +100,13 @@ public class HadoopTaskInfo implements Externalizable {
taskNum = in.readInt();
attempt = in.readInt();
inputSplit = (HadoopInputSplit)in.readObject();
+
+ if (in.readBoolean()) {
+ mapperIdxSet = true;
+ mapperIdx = in.readInt();
+ }
+ else
+ mapperIdxSet = false;
}
/**
@@ -118,6 +138,29 @@ public class HadoopTaskInfo implements Externalizable {
}
/**
+ * @param mapperIdx Current mapper index.
+ */
+ public void mapperIndex(int mapperIdx) {
+ this.mapperIdx = mapperIdx;
+
+ mapperIdxSet = true;
+ }
+
+ /**
+ * @return Current mapper index or {@code null}
+ */
+ public int mapperIndex() {
+ return mapperIdx;
+ }
+
+ /**
+ * @return {@code True} if mapped index is set.
+ */
+ public boolean hasMapperIndex() {
+ return mapperIdxSet;
+ }
+
+ /**
* @return Input split.
*/
@Nullable public HadoopInputSplit inputSplit() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
new file mode 100644
index 0000000..e81dc5f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Direct shuffle message.
+ */
+public class HadoopDirectShuffleMessage implements Message, HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** */
+ @GridToStringInclude
+ private int reducer;
+
+ /** Count. */
+ private int cnt;
+
+ /** Buffer. */
+ private byte[] buf;
+
+ /** Buffer length (equal or less than buf.length). */
+ @GridDirectTransient
+ private transient int bufLen;
+
+ /**
+ * Default constructor.
+ */
+ public HadoopDirectShuffleMessage() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param jobId Job ID.
+ * @param reducer Reducer.
+ * @param cnt Count.
+ * @param buf Buffer.
+ * @param bufLen Buffer length.
+ */
+ public HadoopDirectShuffleMessage(HadoopJobId jobId, int reducer, int cnt, byte[] buf, int bufLen) {
+ assert jobId != null;
+
+ this.jobId = jobId;
+ this.reducer = reducer;
+ this.cnt = cnt;
+ this.buf = buf;
+ this.bufLen = bufLen;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Reducer.
+ */
+ public int reducer() {
+ return reducer;
+ }
+
+ /**
+ * @return Count.
+ */
+ public int count() {
+ return cnt;
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public byte[] buffer() {
+ return buf;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeMessage("jobId", jobId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeInt("reducer", reducer))
+ return false;
+
+ writer.incrementState();
+
+ case 2:
+ if (!writer.writeInt("cnt", cnt))
+ return false;
+
+ writer.incrementState();
+
+ case 3:
+ if (!writer.writeByteArray("buf", this.buf, 0, bufLen))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ jobId = reader.readMessage("jobId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ reducer = reader.readInt("reducer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 2:
+ cnt = reader.readInt("cnt");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 3:
+ this.buf = reader.readByteArray("buf");
+
+ if (!reader.isLastRead())
+ return false;
+
+ bufLen = this.buf != null ? this.buf.length : 0;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(HadoopDirectShuffleMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -42;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+
+ out.writeInt(reducer);
+ out.writeInt(cnt);
+
+ U.writeByteArray(out, buf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+ jobId.readExternal(in);
+
+ reducer = in.readInt();
+ cnt = in.readInt();
+
+ buf = U.readByteArray(in);
+ bufLen = buf != null ? buf.length : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopDirectShuffleMessage.class, this);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java
new file mode 100644
index 0000000..87adcb7
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopMapperUtils.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Set of mapper utility methods.
+ */
+public class HadoopMapperUtils {
+ /** Thread-local mapper index. */
+ private static final ThreadLocal<Integer> MAP_IDX = new ThreadLocal<>();
+
+ /**
+ * @return Current mapper index.
+ */
+ public static int mapperIndex() {
+ Integer res = MAP_IDX.get();
+
+ return res != null ? res : -1;
+ }
+
+ /**
+ * @param idx Current mapper index.
+ */
+ public static void mapperIndex(Integer idx) {
+ MAP_IDX.set(idx);
+ }
+
+ /**
+ * Clear mapper index.
+ */
+ public static void clearMapperIndex() {
+ MAP_IDX.remove();
+ }
+
+ /**
+ * Constructor.
+ */
+ private HadoopMapperUtils() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
index 90a1bad..eec0636 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Context.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperAwareTaskOutput;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
@@ -153,6 +154,16 @@ public class HadoopV2Context extends JobContextImpl implements MapContext, Reduc
}
}
+ /**
+ * Callback invoked from mapper thread when map is finished.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void onMapperFinished() throws IgniteCheckedException {
+ if (output instanceof HadoopMapperAwareTaskOutput)
+ ((HadoopMapperAwareTaskOutput)output).onMapperFinished();
+ }
+
/** {@inheritDoc} */
@Override public OutputCommitter getOutputCommitter() {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
index 418df4e..eb3b935 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2MapTask.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.processors.hadoop.HadoopJobInfo;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
/**
@@ -49,6 +50,11 @@ public class HadoopV2MapTask extends HadoopV2Task {
JobContextImpl jobCtx = taskCtx.jobContext();
+ if (taskCtx.taskInfo().hasMapperIndex())
+ HadoopMapperUtils.mapperIndex(taskCtx.taskInfo().mapperIndex());
+ else
+ HadoopMapperUtils.clearMapperIndex();
+
try {
InputSplit nativeSplit = hadoopContext().getInputSplit();
@@ -72,6 +78,8 @@ public class HadoopV2MapTask extends HadoopV2Task {
try {
mapper.run(new WrappedMapper().getMapContext(hadoopContext()));
+
+ hadoopContext().onMapperFinished();
}
finally {
closeWriter();
@@ -92,6 +100,8 @@ public class HadoopV2MapTask extends HadoopV2Task {
throw new IgniteCheckedException(e);
}
finally {
+ HadoopMapperUtils.clearMapperIndex();
+
if (err != null)
abort(outputFormat);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index 36782bf..a725534 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -1018,6 +1018,8 @@ public class HadoopJobTracker extends HadoopComponent {
if (state == null)
state = initState(jobId);
+ int mapperIdx = 0;
+
for (HadoopInputSplit split : mappers) {
if (state.addMapper(split)) {
if (log.isDebugEnabled())
@@ -1026,6 +1028,8 @@ public class HadoopJobTracker extends HadoopComponent {
HadoopTaskInfo taskInfo = new HadoopTaskInfo(MAP, jobId, meta.taskNumber(split), 0, split);
+ taskInfo.mapperIndex(mapperIdx++);
+
if (tasks == null)
tasks = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 82bbd32..8ffea8c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.processors.hadoop.HadoopComponent;
import org.apache.ignite.internal.processors.hadoop.HadoopContext;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
@@ -39,6 +40,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.plugin.extensions.communication.Message;
+import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -102,8 +104,8 @@ public class HadoopShuffle extends HadoopComponent {
private HadoopShuffleJob<UUID> newJob(HadoopJobId jobId) throws IgniteCheckedException {
HadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
- HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log,
- ctx.jobTracker().job(jobId, null), mem, plan.reducers(), plan.reducers(ctx.localNodeId()), true);
+ HadoopShuffleJob<UUID> job = new HadoopShuffleJob<>(ctx.localNodeId(), log, ctx.jobTracker().job(jobId, null),
+ mem, plan.reducers(), plan.reducers(ctx.localNodeId()), localMappersCount(plan), true);
UUID[] rdcAddrs = new UUID[plan.reducers()];
@@ -123,6 +125,18 @@ public class HadoopShuffle extends HadoopComponent {
}
/**
+ * Get number of local mappers.
+ *
+ * @param plan Plan.
+ * @return Number of local mappers.
+ */
+ private int localMappersCount(HadoopMapReducePlan plan) {
+ Collection<HadoopInputSplit> locMappers = plan.mappers(ctx.localNodeId());
+
+ return F.isEmpty(locMappers) ? 0 : locMappers.size();
+ }
+
+ /**
* @param nodeId Node ID to send message to.
* @param msg Message to send.
* @throws IgniteCheckedException If send failed.
@@ -195,6 +209,11 @@ public class HadoopShuffle extends HadoopComponent {
job(m.jobId()).onShuffleMessage(src, m);
}
+ else if (msg instanceof HadoopDirectShuffleMessage) {
+ HadoopDirectShuffleMessage m = (HadoopDirectShuffleMessage)msg;
+
+ job(m.jobId()).onDirectShuffleMessage(src, m);
+ }
else if (msg instanceof HadoopShuffleAck) {
HadoopShuffleAck m = (HadoopShuffleAck)msg;
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 0a3a0ae..214a335 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -17,20 +17,16 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicReferenceArray;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.processors.hadoop.HadoopJob;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperAwareTaskOutput;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapperUtils;
import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
+import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
@@ -41,6 +37,9 @@ import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopConcurrentHashMultimap;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopSkipList;
+import org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataInput;
+import org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataOutputContext;
+import org.apache.ignite.internal.processors.hadoop.shuffle.direct.HadoopDirectDataOutputState;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -55,9 +54,19 @@ import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReferenceArray;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -121,6 +130,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/** Message size. */
private final int msgSize;
+ /** Whether to strip mappers for remote execution. */
+ private final boolean stripeMappers;
+
/** Local shuffle states. */
private volatile HashMap<T, HadoopShuffleLocalState> locShuffleStates = new HashMap<>();
@@ -143,11 +155,12 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
* @param mem Memory.
* @param totalReducerCnt Amount of reducers in the Job.
* @param locReducers Reducers will work on current node.
+ * @param locMappersCnt Number of mappers running on the given node.
* @param embedded Whether shuffle is running in embedded mode.
* @throws IgniteCheckedException If error.
*/
public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
- int totalReducerCnt, int[] locReducers, boolean embedded) throws IgniteCheckedException {
+ int totalReducerCnt, int[] locReducers, int locMappersCnt, boolean embedded) throws IgniteCheckedException {
this.locReduceAddr = locReduceAddr;
this.totalReducerCnt = totalReducerCnt;
this.job = job;
@@ -155,6 +168,27 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
this.log = log.getLogger(HadoopShuffleJob.class);
this.embedded = embedded;
+ // No stripes for combiner.
+ boolean stripeMappers0 = get(job.info(), SHUFFLE_MAPPER_STRIPED_OUTPUT, false);
+
+ if (stripeMappers0) {
+ if (job.info().hasCombiner()) {
+ log.info("Striped mapper output is disabled because it cannot be used together with combiner [jobId=" +
+ job.id() + ']');
+
+ stripeMappers0 = false;
+ }
+
+ if (!embedded) {
+ log.info("Striped mapper output is disabled becuase it cannot be used in external mode [jobId=" +
+ job.id() + ']');
+
+ stripeMappers0 = false;
+ }
+ }
+
+ stripeMappers = stripeMappers0;
+
msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
locReducersCtx = new AtomicReferenceArray<>(totalReducerCnt);
@@ -169,9 +203,20 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
needPartitioner = totalReducerCnt > 1;
+ // Size of local map is always equal to total reducer number to allow index-based lookup.
locMaps = new AtomicReferenceArray<>(totalReducerCnt);
- rmtMaps = new AtomicReferenceArray<>(totalReducerCnt);
- msgs = new HadoopShuffleMessage[totalReducerCnt];
+
+ // Size of remote map:
+ // - If there are no local mappers, then we will not send anything, so set to 0;
+ // - If output is not striped, then match it to total reducer count, the same way as for local maps.
+ // - If output is striped, then multiply previous value by number of local mappers.
+ int rmtMapsSize = locMappersCnt == 0 ? 0 : totalReducerCnt;
+
+ if (stripeMappers)
+ rmtMapsSize *= locMappersCnt;
+
+ rmtMaps = new AtomicReferenceArray<>(rmtMapsSize);
+ msgs = new HadoopShuffleMessage[rmtMapsSize];
throttle = get(job.info(), SHUFFLE_JOB_THROTTLE, 0);
}
@@ -208,24 +253,26 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
this.io = io;
- if (!flushed) {
- snd = new GridWorker(gridName, "hadoop-shuffle-" + job.id(), log) {
- @Override protected void body() throws InterruptedException {
- try {
- while (!isCancelled()) {
- if (throttle > 0)
- Thread.sleep(throttle);
-
- collectUpdatesAndSend(false);
+ if (!stripeMappers) {
+ if (!flushed) {
+ snd = new GridWorker(gridName, "hadoop-shuffle-" + job.id(), log) {
+ @Override protected void body() throws InterruptedException {
+ try {
+ while (!isCancelled()) {
+ if (throttle > 0)
+ Thread.sleep(throttle);
+
+ collectUpdatesAndSend(false);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IllegalStateException(e);
}
}
- catch (IgniteCheckedException e) {
- throw new IllegalStateException(e);
- }
- }
- };
+ };
- new IgniteThread(snd).start();
+ new IgniteThread(snd).start();
+ }
}
ioInitLatch.countDown();
@@ -306,6 +353,46 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
/**
+ * Process shuffle message.
+ *
+ * @param src Source.
+ * @param msg Message.
+ * @throws IgniteCheckedException Exception.
+ */
+ public void onDirectShuffleMessage(T src, HadoopDirectShuffleMessage msg) throws IgniteCheckedException {
+ assert msg.buffer() != null;
+
+ HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
+
+ HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
+
+ perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
+
+ HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
+
+ HadoopSerialization keySer = taskCtx.keySerialization();
+ HadoopSerialization valSer = taskCtx.valueSerialization();
+
+ // Add data from message to the map.
+ try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
+ HadoopDirectDataInput in = new HadoopDirectDataInput(msg.buffer());
+
+ Object key = null;
+ Object val = null;
+
+ for (int i = 0; i < msg.count(); i++) {
+ key = keySer.read(in, key);
+ val = valSer.read(in, val);
+
+ adder.write(key, val);
+ }
+ }
+
+ if (localShuffleState(src).onShuffleMessage())
+ sendFinishResponse(src, msg.jobId());
+ }
+
+ /**
* @param ack Shuffle ack.
*/
@SuppressWarnings("ConstantConditions")
@@ -467,88 +554,149 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
/**
- * Sends map updates to remote reducers.
+ * Send updates to remote reducers.
+ *
+ * @param flush Flush flag.
+ * @throws IgniteCheckedException If failed.
*/
private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException {
- for (int i = 0; i < rmtMaps.length(); i++) {
- HadoopMultimap map = rmtMaps.get(i);
+ for (int i = 0; i < rmtMaps.length(); i++)
+ collectUpdatesAndSend(i, flush);
+ }
+
+ /**
+ * Send updates to concrete remote reducer.
+ *
+ * @param rmtMapIdx Remote map index.
+ * @param flush Flush flag.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void collectUpdatesAndSend(int rmtMapIdx, boolean flush) throws IgniteCheckedException {
+ final int rmtRdcIdx = stripeMappers ? rmtMapIdx % totalReducerCnt : rmtMapIdx;
- if (map == null)
- continue; // Skip empty map and local node.
+ HadoopMultimap map = rmtMaps.get(rmtMapIdx);
- if (msgs[i] == null)
- msgs[i] = new HadoopShuffleMessage(job.id(), i, msgSize);
+ if (map == null)
+ return;
- final int idx = i;
+ if (msgs[rmtMapIdx] == null)
+ msgs[rmtMapIdx] = new HadoopShuffleMessage(job.id(), rmtRdcIdx, msgSize);
- map.visit(false, new HadoopMultimap.Visitor() {
- /** */
- private long keyPtr;
+ visit(map, rmtMapIdx, rmtRdcIdx);
- /** */
- private int keySize;
+ if (flush && msgs[rmtMapIdx].offset() != 0)
+ send(rmtMapIdx, rmtRdcIdx, 0);
+ }
- /** */
- private boolean keyAdded;
+ /**
+ * Flush remote direct context.
+ *
+ * @param rmtMapIdx Remote map index.
+ * @param rmtDirectCtx Remote direct context.
+ * @param reset Whether to perform reset.
+ */
+ private void sendShuffleMessage(int rmtMapIdx, @Nullable HadoopDirectDataOutputContext rmtDirectCtx, boolean reset) {
+ if (rmtDirectCtx == null)
+ return;
- /** {@inheritDoc} */
- @Override public void onKey(long keyPtr, int keySize) {
- this.keyPtr = keyPtr;
- this.keySize = keySize;
+ int cnt = rmtDirectCtx.count();
- keyAdded = false;
- }
+ if (cnt == 0)
+ return;
- private boolean tryAdd(long valPtr, int valSize) {
- HadoopShuffleMessage msg = msgs[idx];
+ int rmtRdcIdx = stripeMappers ? rmtMapIdx % totalReducerCnt : rmtMapIdx;
- if (!keyAdded) { // Add key and value.
- int size = keySize + valSize;
+ HadoopDirectDataOutputState state = rmtDirectCtx.state();
- if (!msg.available(size, false))
- return false;
+ if (reset)
+ rmtDirectCtx.reset();
- msg.addKey(keyPtr, keySize);
- msg.addValue(valPtr, valSize);
+ HadoopDirectShuffleMessage msg = new HadoopDirectShuffleMessage(job.id(), rmtRdcIdx, cnt,
+ state.buffer(), state.bufferLength());
- keyAdded = true;
+ T nodeId = reduceAddrs[rmtRdcIdx];
- return true;
- }
+ io.apply(nodeId, msg);
+
+ remoteShuffleState(nodeId).onShuffleMessage();
+ }
+
+ /**
+ * Visit output map.
+ *
+ * @param map Map.
+ * @param rmtMapIdx Remote map index.
+ * @param rmtRdcIdx Remote reducer index.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void visit(HadoopMultimap map, final int rmtMapIdx, final int rmtRdcIdx) throws IgniteCheckedException {
+ map.visit(false, new HadoopMultimap.Visitor() {
+ /** */
+ private long keyPtr;
- if (!msg.available(valSize, true))
+ /** */
+ private int keySize;
+
+ /** */
+ private boolean keyAdded;
+
+ /** {@inheritDoc} */
+ @Override public void onKey(long keyPtr, int keySize) {
+ this.keyPtr = keyPtr;
+ this.keySize = keySize;
+
+ keyAdded = false;
+ }
+
+ private boolean tryAdd(long valPtr, int valSize) {
+ HadoopShuffleMessage msg = msgs[rmtMapIdx];
+
+ if (!keyAdded) { // Add key and value.
+ int size = keySize + valSize;
+
+ if (!msg.available(size, false))
return false;
+ msg.addKey(keyPtr, keySize);
msg.addValue(valPtr, valSize);
+ keyAdded = true;
+
return true;
}
- /** {@inheritDoc} */
- @Override public void onValue(long valPtr, int valSize) {
- if (tryAdd(valPtr, valSize))
- return;
+ if (!msg.available(valSize, true))
+ return false;
- send(idx, keySize + valSize);
+ msg.addValue(valPtr, valSize);
- keyAdded = false;
+ return true;
+ }
- if (!tryAdd(valPtr, valSize))
- throw new IllegalStateException();
- }
- });
+ /** {@inheritDoc} */
+ @Override public void onValue(long valPtr, int valSize) {
+ if (tryAdd(valPtr, valSize))
+ return;
- if (flush && msgs[i].offset() != 0)
- send(i, 0);
- }
+ send(rmtMapIdx, rmtRdcIdx, keySize + valSize);
+
+ keyAdded = false;
+
+ if (!tryAdd(valPtr, valSize))
+ throw new IllegalStateException();
+ }
+ });
}
/**
- * @param idx Index of message.
+ * Send message.
+ *
+ * @param rmtMapIdx Remote map index.
+ * @param rmtRdcIdx Remote reducer index.
* @param newBufMinSize Min new buffer size.
*/
- private void send(final int idx, int newBufMinSize) {
- HadoopShuffleMessage msg = msgs[idx];
+ private void send(int rmtMapIdx, int rmtRdcIdx, int newBufMinSize) {
+ HadoopShuffleMessage msg = msgs[rmtMapIdx];
final long msgId = msg.id();
@@ -566,10 +714,10 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
try {
- io.apply(reduceAddrs[idx], msg);
+ io.apply(reduceAddrs[rmtRdcIdx], msg);
if (embedded)
- remoteShuffleState(reduceAddrs[idx]).onShuffleMessage();
+ remoteShuffleState(reduceAddrs[rmtRdcIdx]).onShuffleMessage();
}
catch (GridClosureException e) {
if (fut != null)
@@ -593,7 +741,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
});
}
- msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
+ msgs[rmtMapIdx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), rmtRdcIdx,
Math.max(msgSize, newBufMinSize));
}
@@ -639,31 +787,33 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
if (totalReducerCnt == 0)
return new GridFinishedFuture<>();
- U.await(ioInitLatch);
+ if (!stripeMappers) {
+ U.await(ioInitLatch);
- GridWorker snd0 = snd;
+ GridWorker snd0 = snd;
- if (snd0 != null) {
- if (log.isDebugEnabled())
- log.debug("Cancelling sender thread.");
+ if (snd0 != null) {
+ if (log.isDebugEnabled())
+ log.debug("Cancelling sender thread.");
- snd0.cancel();
+ snd0.cancel();
- try {
- snd0.join();
+ try {
+ snd0.join();
- if (log.isDebugEnabled())
- log.debug("Finished waiting for sending thread to complete on shuffle job flush: " + job.id());
- }
- catch (InterruptedException e) {
- throw new IgniteInterruptedCheckedException(e);
+ if (log.isDebugEnabled())
+ log.debug("Finished waiting for sending thread to complete on shuffle job flush: " + job.id());
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedCheckedException(e);
+ }
}
- }
- collectUpdatesAndSend(true); // With flush.
+ collectUpdatesAndSend(true); // With flush.
- if (log.isDebugEnabled())
- log.debug("Finished sending collected updates to remote reducers: " + job.id());
+ if (log.isDebugEnabled())
+ log.debug("Finished sending collected updates to remote reducers: " + job.id());
+ }
GridCompoundFuture fut = new GridCompoundFuture<>();
@@ -700,8 +850,8 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
if (log.isDebugEnabled())
log.debug("Collected futures to compound futures for flush: " + sentMsgs.size());
-
}
+
return fut;
}
@@ -775,13 +925,17 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/**
* Partitioned output.
*/
- private class PartitionedOutput implements HadoopTaskOutput {
+ public class PartitionedOutput implements HadoopMapperAwareTaskOutput {
/** */
private final HadoopTaskOutput[] locAdders = new HadoopTaskOutput[locMaps.length()];
/** */
private final HadoopTaskOutput[] rmtAdders = new HadoopTaskOutput[rmtMaps.length()];
+ /** Remote direct contexts. */
+ private final HadoopDirectDataOutputContext[] rmtDirectCtxs =
+ new HadoopDirectDataOutputContext[rmtMaps.length()];
+
/** */
private HadoopPartitioner partitioner;
@@ -819,16 +973,53 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
locAdders[part] = out = getOrCreateMap(locMaps, part).startAdding(taskCtx);
}
else {
- out = rmtAdders[part];
+ if (stripeMappers) {
+ int mapperIdx = HadoopMapperUtils.mapperIndex();
- if (out == null)
- rmtAdders[part] = out = getOrCreateMap(rmtMaps, part).startAdding(taskCtx);
+ assert mapperIdx >= 0;
+
+ int idx = totalReducerCnt * mapperIdx + part;
+
+ HadoopDirectDataOutputContext rmtDirectCtx = rmtDirectCtxs[idx];
+
+ if (rmtDirectCtx == null) {
+ rmtDirectCtx = new HadoopDirectDataOutputContext(msgSize, taskCtx);
+
+ rmtDirectCtxs[idx] = rmtDirectCtx;
+ }
+
+ if (rmtDirectCtx.write(key, val))
+ sendShuffleMessage(idx, rmtDirectCtx, true);
+
+ return;
+ }
+ else {
+ out = rmtAdders[part];
+
+ if (out == null)
+ rmtAdders[part] = out = getOrCreateMap(rmtMaps, part).startAdding(taskCtx);
+ }
}
out.write(key, val);
}
/** {@inheritDoc} */
+ @Override public void onMapperFinished() throws IgniteCheckedException {
+ if (stripeMappers) {
+ int mapperIdx = HadoopMapperUtils.mapperIndex();
+
+ assert mapperIdx >= 0;
+
+ for (int i = 0; i < totalReducerCnt; i++) {
+ int idx = totalReducerCnt * mapperIdx + i;
+
+ sendShuffleMessage(idx, rmtDirectCtxs[idx], false);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void close() throws IgniteCheckedException {
for (HadoopTaskOutput adder : locAdders) {
if (adder != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
index 5ffaa55..4331124 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleRemoteState.java
@@ -17,17 +17,14 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
-import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.lang.IgniteInClosure;
-import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
/**
* Remote shuffle state.
*/
-class HadoopShuffleRemoteState<T> {
+class HadoopShuffleRemoteState {
/** Message count. */
private final AtomicLong msgCnt = new AtomicLong();
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
new file mode 100644
index 0000000..e3a713a
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataInput.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF;
+
+/**
+ * Hadoop data input used for direct communication.
+ */
+public class HadoopDirectDataInput extends InputStream implements DataInput {
+ /** Data buffer. */
+ private final byte[] buf;
+
+ /** Position. */
+ private int pos;
+
+ /**
+ * Constructor.
+ *
+ * @param buf Buffer.
+ */
+ public HadoopDirectDataInput(byte[] buf) {
+ this.buf = buf;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read() throws IOException {
+ return readByte();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(@NotNull byte[] b) throws IOException {
+ readFully(b, 0, b.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+ System.arraycopy(buf, pos, b, off, len);
+
+ pos += len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int skipBytes(int n) throws IOException {
+ pos += n;
+
+ return n;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readBoolean() throws IOException {
+ return readByte() == 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte readByte() throws IOException {
+ byte res = GridUnsafe.getByte(buf, BYTE_ARR_OFF + pos);
+
+ pos += 1;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int readUnsignedByte() throws IOException {
+ return readByte() & 0xff;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short readShort() throws IOException {
+ short res = GridUnsafe.getShort(buf, BYTE_ARR_OFF + pos);
+
+ pos += 2;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int readUnsignedShort() throws IOException {
+ return readShort() & 0xffff;
+ }
+
+ /** {@inheritDoc} */
+ @Override public char readChar() throws IOException {
+ char res = GridUnsafe.getChar(buf, BYTE_ARR_OFF + pos);
+
+ pos += 2;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int readInt() throws IOException {
+ int res = GridUnsafe.getInt(buf, BYTE_ARR_OFF + pos);
+
+ pos += 4;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long readLong() throws IOException {
+ long res = GridUnsafe.getLong(buf, BYTE_ARR_OFF + pos);
+
+ pos += 8;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float readFloat() throws IOException {
+ float res = GridUnsafe.getFloat(buf, BYTE_ARR_OFF + pos);
+
+ pos += 4;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public double readDouble() throws IOException {
+ double res = GridUnsafe.getDouble(buf, BYTE_ARR_OFF + pos);
+
+ pos += 8;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String readLine() throws IOException {
+ // TODO: Create ticket!
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @NotNull @Override public String readUTF() throws IOException {
+ byte[] bytes = new byte[readShort()];
+
+ if (bytes.length != 0)
+ readFully(bytes);
+
+ return new String(bytes, StandardCharsets.UTF_8);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
new file mode 100644
index 0000000..151e552
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutput.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UTFDataFormatException;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF;
+
+/**
+ * Hadoop data output for direct communication.
+ */
+public class HadoopDirectDataOutput extends OutputStream implements DataOutput {
+ /** Flush size. */
+ private final int flushSize;
+
+ /** Data buffer. */
+ private byte[] buf;
+
+ /** Buffer size. */
+ private int bufSize;
+
+ /** Position. */
+ private int pos;
+
+ /**
+ * Constructor.
+ *
+ * @param flushSize Flush size.
+ */
+ public HadoopDirectDataOutput(int flushSize) {
+ this(flushSize, flushSize);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param flushSize Flush size.
+ * @param allocSize Allocation size.
+ */
+ public HadoopDirectDataOutput(int flushSize, int allocSize) {
+ this.flushSize = flushSize;
+
+ buf = new byte[allocSize];
+ bufSize = allocSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(@NotNull byte[] b) throws IOException {
+ write(b, 0, b.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(@NotNull byte[] b, int off, int len) throws IOException {
+ int writePos = ensure(len);
+
+ System.arraycopy(b, off, buf, writePos, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(int val) throws IOException {
+ writeByte(val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBoolean(boolean val) throws IOException {
+ writeByte(val ? (byte)1 : (byte)0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeByte(int val) throws IOException {
+ int writePos = ensure(1);
+
+ buf[writePos] = (byte)val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeShort(int val) throws IOException {
+ int writePos = ensure(2);
+
+ GridUnsafe.putShort(buf, BYTE_ARR_OFF + writePos, (short)val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeChar(int val) throws IOException {
+ int writePos = ensure(2);
+
+ GridUnsafe.putChar(buf, BYTE_ARR_OFF + writePos, (char)val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeInt(int val) throws IOException {
+ int writePos = ensure(4);
+
+ GridUnsafe.putInt(buf, BYTE_ARR_OFF + writePos, val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeLong(long val) throws IOException {
+ int writePos = ensure(8);
+
+ GridUnsafe.putLong(buf, BYTE_ARR_OFF + writePos, val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeFloat(float val) throws IOException {
+ int writePos = ensure(4);
+
+ GridUnsafe.putFloat(buf, BYTE_ARR_OFF + writePos, val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeDouble(double val) throws IOException {
+ int writePos = ensure(8);
+
+ GridUnsafe.putDouble(buf, BYTE_ARR_OFF + writePos, val);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeBytes(@NotNull String str) throws IOException {
+ for(int i = 0; i < str.length(); ++i)
+ write((byte)str.charAt(i));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeChars(@NotNull String str) throws IOException {
+ for (int i = 0; i < str.length(); ++i)
+ writeChar(str.charAt(i));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeUTF(@NotNull String str) throws IOException {
+ byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
+
+ int len = bytes.length;
+
+ if (len > 65535)
+ throw new UTFDataFormatException("UTF8 form of string is longer than 65535 bytes: " + str);
+
+ writeShort((short)len);
+ write(bytes);
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public byte[] buffer() {
+ return buf;
+ }
+
+ /**
+ * @return Position.
+ */
+ public int position() {
+ return pos;
+ }
+
+ /**
+ * @return Whether buffer is ready for flush.
+ */
+ public boolean readyForFlush() {
+ return pos >= flushSize;
+ }
+
+ /**
+ * Ensure that the given amount of bytes is available within the stream, then shift the position.
+ *
+ * @param cnt Count.
+ * @return Position
+ */
+ private int ensure(int cnt) {
+ int pos0 = pos;
+
+ if (pos0 + cnt > bufSize)
+ grow(pos0 + cnt);
+
+ pos += cnt;
+
+ return pos0;
+ }
+
+ /**
+ * Grow array up to the given count.
+ *
+ * @param cnt Count.
+ */
+ private void grow(int cnt) {
+ int bufSize0 = (int)(bufSize * 1.1);
+
+ if (bufSize0 < cnt)
+ bufSize0 = cnt;
+
+ byte[] buf0 = new byte[bufSize0];
+
+ System.arraycopy(buf, 0, buf0, 0, pos);
+
+ buf = buf0;
+ bufSize = bufSize0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
new file mode 100644
index 0000000..bc70ef3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputContext.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
+import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
+
+/**
+ * Hadoop data output context for direct communication.
+ */
+public class HadoopDirectDataOutputContext {
+ /** Flush size. */
+ private final int flushSize;
+
+ /** Key serialization. */
+ private final HadoopSerialization keySer;
+
+ /** Value serialization. */
+ private final HadoopSerialization valSer;
+
+ /** Data output. */
+ private HadoopDirectDataOutput out;
+
+ /** Number of keys written. */
+ private int cnt;
+
+ /**
+ * Constructor.
+ *
+ * @param flushSize Flush size.
+ * @param taskCtx Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopDirectDataOutputContext(int flushSize, HadoopTaskContext taskCtx)
+ throws IgniteCheckedException {
+ this.flushSize = flushSize;
+
+ keySer = taskCtx.keySerialization();
+ valSer = taskCtx.valueSerialization();
+
+ out = new HadoopDirectDataOutput(flushSize);
+ }
+
+ /**
+ * Write key-value pair.
+ *
+ * @param key Key.
+ * @param val Value.
+ * @return Whether flush is needed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean write(Object key, Object val) throws IgniteCheckedException {
+ keySer.write(out, key);
+ valSer.write(out, val);
+
+ cnt++;
+
+ return out.readyForFlush();
+ }
+
+ /**
+ * @return Key-value pairs count.
+ */
+ public int count() {
+ return cnt;
+ }
+
+ /**
+ * @return State.
+ */
+ public HadoopDirectDataOutputState state() {
+ return new HadoopDirectDataOutputState(out.buffer(), out.position());
+ }
+
+ /**
+ * Reset buffer.
+ */
+ public void reset() {
+ int allocSize = Math.max(flushSize, out.position());
+
+ out = new HadoopDirectDataOutput(flushSize, allocSize);
+ cnt = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
new file mode 100644
index 0000000..a9c12e3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/direct/HadoopDirectDataOutputState.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle.direct;
+
+/**
+ * Hadoop data output state for direct communication.
+ */
+public class HadoopDirectDataOutputState {
+ /** Buffer. */
+ private final byte[] buf;
+
+ /** Buffer length. */
+ private final int bufLen;
+
+ /**
+ * Constructor.
+ *
+ * @param buf Buffer.
+ * @param bufLen Buffer length.
+ */
+ public HadoopDirectDataOutputState(byte[] buf, int bufLen) {
+ this.buf = buf;
+ this.bufLen = bufLen;
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public byte[] buffer() {
+ return buf;
+ }
+
+ /**
+ * @return Length.
+ */
+ public int bufferLength() {
+ return bufLen;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
index cb08c00..3336120 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -151,7 +151,7 @@ public class HadoopChildProcessRunner {
job.initialize(true, nodeDesc.processId());
shuffleJob = new HadoopShuffleJob<>(comm.localProcessDescriptor(), log, job, mem,
- req.totalReducerCount(), req.localReducers(), false);
+ req.totalReducerCount(), req.localReducers(), 0, false);
initializeExecutors(req);
http://git-wip-us.apache.org/repos/asf/ignite/blob/a976c425/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
index b04deeb..8897a38 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopMapReduceEmbeddedSelfTest.java
@@ -34,6 +34,7 @@ import org.apache.ignite.configuration.HadoopConfiguration;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount1;
import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount2;
@@ -54,12 +55,28 @@ public class HadoopMapReduceEmbeddedSelfTest extends HadoopMapReduceTest {
return cfg;
}
+ /*
+ * @throws Exception If fails.
+ */
+ public void testMultiReducerWholeMapReduceExecution() throws Exception {
+ checkMultiReducerWholeMapReduceExecution(false);
+ }
+
+ /*
+ * @throws Exception If fails.
+ */
+ public void testMultiReducerWholeMapReduceExecutionStriped() throws Exception {
+ checkMultiReducerWholeMapReduceExecution(true);
+ }
+
/**
* Tests whole job execution with all phases in old and new versions of API with definition of custom
* Serialization, Partitioner and IO formats.
+ *
+ * @param striped Whether output should be striped or not.
* @throws Exception If fails.
*/
- public void testMultiReducerWholeMapReduceExecution() throws Exception {
+ public void checkMultiReducerWholeMapReduceExecution(boolean striped) throws Exception {
IgfsPath inDir = new IgfsPath(PATH_INPUT);
igfs.mkdirs(inDir);
@@ -81,6 +98,9 @@ public class HadoopMapReduceEmbeddedSelfTest extends HadoopMapReduceTest {
JobConf jobConf = new JobConf();
+ if (striped)
+ jobConf.set(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), "true");
+
jobConf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
//To split into about 6-7 items for v2
[28/50] [abbrv] ignite git commit: IGNITE-4277: Hadoop: implemented
"partially raw" comparator. This closes #1345.
Posted by sb...@apache.org.
IGNITE-4277: Hadoop: implemented "partially raw" comparator. This closes #1345.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c1ddf21f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c1ddf21f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c1ddf21f
Branch: refs/heads/master
Commit: c1ddf21fd627c76a8b7e0d81ad43480b1f1e204d
Parents: 30b869d
Author: devozerov <vo...@gridgain.com>
Authored: Thu Dec 15 11:58:28 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:41 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 1 +
.../processors/hadoop/HadoopJobProperty.java | 6 +-
.../processors/hadoop/HadoopTaskContext.java | 8 ++
.../io/PartiallyOffheapRawComparatorEx.java | 33 +++++
.../hadoop/io/PartiallyRawComparator.java | 33 +++++
.../org/apache/ignite/hadoop/io/RawMemory.java | 86 ++++++++++++
.../hadoop/io/TextPartiallyRawComparator.java | 115 ++++++++++++++++
.../apache/ignite/hadoop/io/package-info.java | 22 ++++
...DelegatingPartiallyOffheapRawComparator.java | 54 ++++++++
.../hadoop/impl/v2/HadoopV2TaskContext.java | 21 +++
.../processors/hadoop/io/OffheapRawMemory.java | 131 +++++++++++++++++++
.../shuffle/collections/HadoopSkipList.java | 14 +-
.../hadoop/impl/HadoopTeraSortTest.java | 7 +
.../collections/HadoopAbstractMapTest.java | 6 +
14 files changed, 535 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index f6c2fa9..81c1405 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -372,6 +372,7 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
// We use "contains" instead of "equals" to handle subclasses properly.
if (clsName.contains("org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem") ||
clsName.contains("org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem") ||
+ clsName.contains("org.apache.ignite.hadoop.io.TextPartialRawComparator") ||
clsName.contains("org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider"))
return true;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 9e1dede..4122eef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -64,6 +64,11 @@ public enum HadoopJobProperty {
JOB_SHARED_CLASSLOADER("ignite.job.shared.classloader"),
/**
+ * Fully qualified name of partially-raw comparator which should be used on sorting phase.
+ */
+ JOB_PARTIAL_RAW_COMPARATOR("ignite.job.partial.raw.comparator"),
+
+ /**
* Size in bytes of single memory page which will be allocated for data structures in shuffle.
* <p>
* By default is {@code 32 * 1024}.
@@ -112,7 +117,6 @@ public enum HadoopJobProperty {
*/
SHUFFLE_JOB_THROTTLE("ignite.shuffle.job.throttle");
-
/** Property name. */
private final String propName;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
index ecb9f26..dddd017 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskContext.java
@@ -22,6 +22,7 @@ import java.util.concurrent.Callable;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounter;
import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
/**
* Task context.
@@ -157,6 +158,13 @@ public abstract class HadoopTaskContext {
public abstract Comparator<Object> sortComparator();
/**
+ * Get semi-raw sorting comparator.
+ *
+ * @return Semi-raw sorting comparator.
+ */
+ public abstract PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator();
+
+ /**
* Gets comparator for grouping on combine or reduce operation.
*
* @return Comparator.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java
new file mode 100644
index 0000000..157609e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/io/PartiallyOffheapRawComparatorEx.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.io;
+
+/**
+ * Special version of raw comparator allowing direct access to the underlying memory.
+ */
+public interface PartiallyOffheapRawComparatorEx<T> {
+ /**
+ * Perform compare.
+ *
+ * @param val1 First value.
+ * @param val2Ptr Pointer to the second value data.
+ * @param val2Len Length of the second value data.
+ * @return Result.
+ */
+ int compare(T val1, long val2Ptr, int val2Len);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java
new file mode 100644
index 0000000..b9a4505
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/PartiallyRawComparator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+/**
+ * Partially raw comparator. Compares one deserialized value with serialized value.
+ */
+public interface PartiallyRawComparator<T> {
+ /**
+ * Do compare.
+ *
+ * @param val1 First value (deserialized).
+ * @param val2Buf Second value (serialized).
+ * @return A negative integer, zero, or a positive integer as this object is less than, equal to, or greater
+ * than the specified object.
+ */
+ int compare(T val1, RawMemory val2Buf);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java
new file mode 100644
index 0000000..8dcaf83
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/RawMemory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+/**
+ * Memory abstraction for raw comparison.
+ */
+public interface RawMemory {
+ /**
+ * Get byte value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ byte get(int idx);
+
+ /**
+ * Get short value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ short getShort(int idx);
+
+ /**
+ * Get char value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ char getChar(int idx);
+
+ /**
+ * Get int value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ int getInt(int idx);
+
+ /**
+ * Get long value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ long getLong(int idx);
+
+ /**
+ * Get float value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ float getFloat(int idx);
+
+ /**
+ * Get double value at the given index.
+ *
+ * @param idx Index.
+ * @return Value.
+ */
+ double getDouble(int idx);
+
+ /**
+ * Get length.
+ *
+ * @return Length.
+ */
+ int length();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
new file mode 100644
index 0000000..a2bc3d4
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/TextPartiallyRawComparator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.io;
+
+import com.google.common.primitives.Longs;
+import com.google.common.primitives.UnsignedBytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.ignite.internal.processors.hadoop.io.OffheapRawMemory;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
+import org.apache.ignite.internal.util.GridUnsafe;
+
+/**
+ * Partial raw comparator for {@link Text} data type.
+ * <p>
+ * Implementation is borrowed from {@code org.apache.hadoop.io.FastByteComparisons} and adopted to Ignite
+ * infrastructure.
+ */
+public class TextPartiallyRawComparator implements PartiallyRawComparator<Text>, PartiallyOffheapRawComparatorEx<Text> {
+ /** {@inheritDoc} */
+ @Override public int compare(Text val1, RawMemory val2Buf) {
+ if (val2Buf instanceof OffheapRawMemory) {
+ OffheapRawMemory val2Buf0 = (OffheapRawMemory)val2Buf;
+
+ return compare(val1, val2Buf0.pointer(), val2Buf0.length());
+ }
+ else
+ throw new UnsupportedOperationException("Text can be compared only with offheap memory.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compare(Text val1, long val2Ptr, int val2Len) {
+ int len2 = WritableUtils.decodeVIntSize(GridUnsafe.getByte(val2Ptr));
+
+ return compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + len2, val2Len - len2);
+ }
+
+ /**
+ * Internal comparison routine.
+ *
+ * @param buf1 Bytes 1.
+ * @param len1 Length 1.
+ * @param ptr2 Pointer 2.
+ * @param len2 Length 2.
+ * @return Result.
+ */
+ @SuppressWarnings("SuspiciousNameCombination")
+ private static int compareBytes(byte[] buf1, int len1, long ptr2, int len2) {
+ int minLength = Math.min(len1, len2);
+
+ int minWords = minLength / Longs.BYTES;
+
+ for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
+ long lw = GridUnsafe.getLong(buf1, GridUnsafe.BYTE_ARR_OFF + i);
+ long rw = GridUnsafe.getLong(ptr2 + i);
+
+ long diff = lw ^ rw;
+
+ if (diff != 0) {
+ if (GridUnsafe.BIG_ENDIAN)
+ return (lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE) ? -1 : 1;
+
+ // Use binary search
+ int n = 0;
+ int y;
+ int x = (int) diff;
+
+ if (x == 0) {
+ x = (int) (diff >>> 32);
+
+ n = 32;
+ }
+
+ y = x << 16;
+
+ if (y == 0)
+ n += 16;
+ else
+ x = y;
+
+ y = x << 8;
+
+ if (y == 0)
+ n += 8;
+
+ return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+ }
+ }
+
+ // The epilogue to cover the last (minLength % 8) elements.
+ for (int i = minWords * Longs.BYTES; i < minLength; i++) {
+ int res = UnsignedBytes.compare(buf1[i], GridUnsafe.getByte(ptr2 + i));
+
+ if (res != 0)
+ return res;
+ }
+
+ return len1 - len2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java
new file mode 100644
index 0000000..0d1f7b9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/io/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains <b>Hadoop Accelerator</b> API for input-output operations.
+ */
+package org.apache.ignite.hadoop.io;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java
new file mode 100644
index 0000000..e6d369e
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2DelegatingPartiallyOffheapRawComparator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.v2;
+
+import org.apache.ignite.hadoop.io.PartiallyRawComparator;
+import org.apache.ignite.internal.processors.hadoop.io.OffheapRawMemory;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
+
+/**
+ * Delegating partial raw comparator.
+ */
+public class HadoopV2DelegatingPartiallyOffheapRawComparator<T> implements PartiallyOffheapRawComparatorEx<T> {
+ /** Target comparator. */
+ private final PartiallyRawComparator<T> target;
+
+ /** Memory. */
+ private OffheapRawMemory mem;
+
+ /**
+ * Constructor.
+ *
+ * @param target Target.
+ */
+ public HadoopV2DelegatingPartiallyOffheapRawComparator(PartiallyRawComparator<T> target) {
+ assert target != null;
+
+ this.target = target;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compare(T val1, long val2Ptr, int val2Len) {
+ if (mem == null)
+ mem = new OffheapRawMemory(val2Ptr, val2Len);
+ else
+ mem.update(val2Ptr, val2Len);
+
+ return target.compare(val1, mem);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index d444f2b..42bbec5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -38,13 +38,16 @@ import org.apache.hadoop.mapreduce.JobSubmissionFiles;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.hadoop.io.PartiallyRawComparator;
import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
import org.apache.ignite.internal.processors.hadoop.HadoopExternalSplit;
import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
import org.apache.ignite.internal.processors.hadoop.HadoopJob;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner;
import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper;
@@ -62,6 +65,7 @@ import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1MapTask;
import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1Partitioner;
import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1ReduceTask;
import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1SetupTask;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.A;
@@ -421,11 +425,28 @@ public class HadoopV2TaskContext extends HadoopTaskContext {
}
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override public Comparator<Object> sortComparator() {
return (Comparator<Object>)jobCtx.getSortComparator();
}
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() {
+ Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(), null);
+
+ if (cls == null)
+ return null;
+
+ Object res = ReflectionUtils.newInstance(cls, jobConf());
+
+ if (res instanceof PartiallyOffheapRawComparatorEx)
+ return (PartiallyOffheapRawComparatorEx)res;
+ else
+ return new HadoopV2DelegatingPartiallyOffheapRawComparator<>((PartiallyRawComparator)res);
+ }
+
+ /** {@inheritDoc} */
@Override public Comparator<Object> groupComparator() {
Comparator<?> res;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java
new file mode 100644
index 0000000..564f92c
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/io/OffheapRawMemory.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.io;
+
+import org.apache.ignite.hadoop.io.RawMemory;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Offheap-based memory.
+ */
+public class OffheapRawMemory implements RawMemory {
+ /** Pointer. */
+ private long ptr;
+
+ /** Length. */
+ private int len;
+
+ /**
+ * Constructor.
+ *
+ * @param ptr Pointer.
+ * @param len Length.
+ */
+ public OffheapRawMemory(long ptr, int len) {
+ update(ptr, len);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte get(int idx) {
+ ensure(idx, 1);
+
+ return GridUnsafe.getByte(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short getShort(int idx) {
+ ensure(idx, 2);
+
+ return GridUnsafe.getShort(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public char getChar(int idx) {
+ ensure(idx, 2);
+
+ return GridUnsafe.getChar(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getInt(int idx) {
+ ensure(idx, 4);
+
+ return GridUnsafe.getInt(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLong(int idx) {
+ ensure(idx, 8);
+
+ return GridUnsafe.getLong(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getFloat(int idx) {
+ ensure(idx, 4);
+
+ return GridUnsafe.getFloat(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public double getDouble(int idx) {
+ ensure(idx, 8);
+
+ return GridUnsafe.getDouble(ptr + idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int length() {
+ return len;
+ }
+
+ /**
+ * @return Raw pointer.
+ */
+ public long pointer() {
+ return ptr;
+ }
+
+ /**
+ * Update pointer and length.
+ *
+ * @param ptr Pointer.
+ * @param len Length.
+ */
+ public void update(long ptr, int len) {
+ this.ptr = ptr;
+ this.len = len;
+ }
+
+ /**
+ * Ensure that the given number of bytes are available for read. Throw an exception otherwise.
+ *
+ * @param idx Index.
+ * @param cnt Count.
+ */
+ private void ensure(int idx, int cnt) {
+ if (idx < 0 || idx + cnt - 1 >= len)
+ throw new IndexOutOfBoundsException("Illegal index [len=" + len + ", idx=" + idx + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(OffheapRawMemory.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
index 7db88bc..f300a18 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopSkipList.java
@@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopJobInfo;
import org.apache.ignite.internal.processors.hadoop.HadoopSerialization;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
@@ -280,6 +281,9 @@ public class HadoopSkipList extends HadoopMultimapBase {
private final Comparator<Object> cmp;
/** */
+ private final PartiallyOffheapRawComparatorEx<Object> partialRawCmp;
+
+ /** */
private final Random rnd = new GridRandom();
/** */
@@ -298,6 +302,7 @@ public class HadoopSkipList extends HadoopMultimapBase {
keyReader = new Reader(keySer);
cmp = ctx.sortComparator();
+ partialRawCmp = ctx.partialRawSortComparator();
}
/** {@inheritDoc} */
@@ -475,7 +480,14 @@ public class HadoopSkipList extends HadoopMultimapBase {
private int cmp(Object key, long meta) {
assert meta != 0;
- return cmp.compare(key, keyReader.readKey(meta));
+ if (partialRawCmp != null) {
+ long keyPtr = key(meta);
+ int keySize = keySize(keyPtr);
+
+ return partialRawCmp.compare(key, keyPtr + 4, keySize);
+ }
+ else
+ return cmp.compare(key, keyReader.readKey(meta));
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
index 0cc9564..a016506 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -41,8 +41,10 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.HadoopConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.hadoop.io.TextPartiallyRawComparator;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
@@ -161,6 +163,11 @@ public class HadoopTeraSortTest extends HadoopAbstractSelfTest {
jobConf.set("mapred.min.split.size", String.valueOf(splitSize));
jobConf.set("mapred.max.split.size", String.valueOf(splitSize));
+ jobConf.setBoolean(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), true);
+
+ jobConf.set(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(),
+ TextPartiallyRawComparator.class.getName());
+
Job job = setupConfig(jobConf);
HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c1ddf21f/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
index 9d1fd4f..1f8978d 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopAbstractMapTest.java
@@ -33,6 +33,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounter;
import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters;
import org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopWritableSerialization;
+import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.Nullable;
@@ -84,6 +85,11 @@ public abstract class HadoopAbstractMapTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
+ @Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public Comparator<Object> groupComparator() {
return ComparableComparator.getInstance();
[48/50] [abbrv] ignite git commit: TcpCommunicationSpi: fixed typo in
warning.
Posted by sb...@apache.org.
TcpCommunicationSpi: fixed typo in warning.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ce46c10e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ce46c10e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ce46c10e
Branch: refs/heads/master
Commit: ce46c10ea767254a60b80abecf45be8a2d9921b4
Parents: e8fd165
Author: sboikov <sb...@gridgain.com>
Authored: Wed Dec 21 13:09:26 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Dec 21 13:09:26 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ce46c10e/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0c90414..c35b5ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -1872,7 +1872,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
if (msgQueueLimit == 0)
U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " +
"potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " +
- "due to message queues growth on sender and reciever sides.");
+ "due to message queues growth on sender and receiver sides.");
registerMBean(gridName, this, TcpCommunicationSpiMBean.class);
[05/50] [abbrv] ignite git commit: IGNITE-4379: Fixed local query
execution. This closes #1323.
Posted by sb...@apache.org.
IGNITE-4379: Fixed local query execution. This closes #1323.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6e485637
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6e485637
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6e485637
Branch: refs/heads/master
Commit: 6e485637e2738a7e809eac1a802f0964dc12383d
Parents: b83ec8e
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Fri Dec 9 15:42:40 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Dec 9 15:42:40 2016 +0300
----------------------------------------------------------------------
.../processors/query/h2/IgniteH2Indexing.java | 54 ++++++++++----------
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 19 +++++++
2 files changed, 45 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e485637/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 6da8758..362ddd8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -774,31 +774,38 @@ public class IgniteH2Indexing implements GridQueryIndexing {
throws IgniteCheckedException {
final Connection conn = connectionForSpace(spaceName);
- initLocalQueryContext(conn, enforceJoinOrder, filters);
+ setupConnection(conn, false, enforceJoinOrder);
+
+ final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
+
+ List<GridQueryFieldMetadata> meta;
try {
- final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
+ meta = meta(stmt.getMetaData());
+ }
+ catch (SQLException e) {
+ throw new IgniteCheckedException("Cannot prepare query metadata", e);
+ }
- List<GridQueryFieldMetadata> meta;
+ final GridH2QueryContext ctx = new GridH2QueryContext(nodeId, nodeId, 0, LOCAL)
+ .filter(filters).distributedJoins(false);
- try {
- meta = meta(stmt.getMetaData());
- }
- catch (SQLException e) {
- throw new IgniteCheckedException("Cannot prepare query metadata", e);
- }
+ return new GridQueryFieldsResultAdapter(meta, null) {
+ @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException {
+ assert GridH2QueryContext.get() == null;
+
+ GridH2QueryContext.set(ctx);
- return new GridQueryFieldsResultAdapter(meta, null) {
- @Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException{
+ try {
ResultSet rs = executeSqlQueryWithTimer(spaceName, stmt, conn, qry, params, timeout, cancel);
return new FieldsIterator(rs);
}
- };
- }
- finally {
- GridH2QueryContext.clearThreadLocal();
- }
+ finally {
+ GridH2QueryContext.clearThreadLocal();
+ }
+ }
+ };
}
/**
@@ -1025,17 +1032,6 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/**
- * @param conn Connection.
- * @param enforceJoinOrder Enforce join order of tables.
- * @param filter Filter.
- */
- private void initLocalQueryContext(Connection conn, boolean enforceJoinOrder, IndexingQueryFilter filter) {
- setupConnection(conn, false, enforceJoinOrder);
-
- GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
- }
-
- /**
* @param conn Connection to use.
* @param distributedJoins If distributed joins are enabled.
* @param enforceJoinOrder Enforce join order of tables.
@@ -1061,7 +1057,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
Connection conn = connectionForThread(tbl.schemaName());
- initLocalQueryContext(conn, false, filter);
+ setupConnection(conn, false, false);
+
+ GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
try {
ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e485637/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
index d5f02eb..2c35501 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
@@ -28,10 +28,12 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
+import javax.cache.Cache;
import javax.cache.CacheException;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.affinity.AffinityKey;
@@ -650,6 +652,23 @@ public abstract class IgniteCacheAbstractFieldsQuerySelfTest extends GridCommonA
}
/** @throws Exception If failed. */
+ public void testLocalQuery() throws Exception {
+ IgniteCache<Object, Object> cache = grid(0).cache( null);
+
+ int expected = 0;
+
+ for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){
+ if(e.getValue() instanceof Integer)
+ expected++;
+ }
+
+ QueryCursor<List<?>> qry = cache
+ .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true));
+
+ assertEquals(expected, qry.getAll().size());
+ }
+
+ /** @throws Exception If failed. */
public void testQueryIntegersWithJoin() throws Exception {
QueryCursor<List<?>> qry = grid(0).cache(null).query(new SqlFieldsQuery(
"select i._KEY, i._VAL, j._KEY, j._VAL from Integer i join Integer j where i._VAL >= 100"));
[20/50] [abbrv] ignite git commit: IGNITE-4355: Hadoop: Implemented
parallel task context initialization during shuffle. This closes #1310. This
closes #1313.
Posted by sb...@apache.org.
IGNITE-4355: Hadoop: Implemented parallel task context initialization during shuffle. This closes #1310. This closes #1313.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/04cff9b5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/04cff9b5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/04cff9b5
Branch: refs/heads/master
Commit: 04cff9b53df1da21c7552b2e47df50cacbae9158
Parents: b7b97cf
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 5 16:28:05 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:45:42 2016 +0300
----------------------------------------------------------------------
.../hadoop/shuffle/HadoopShuffle.java | 23 ++++++---
.../hadoop/shuffle/HadoopShuffleJob.java | 53 ++++++++++++++++++--
2 files changed, 66 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/04cff9b5/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index a69e779..4450bf2 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -53,6 +53,9 @@ public class HadoopShuffle extends HadoopComponent {
/** */
protected final GridUnsafeMemory mem = new GridUnsafeMemory(0);
+ /** Mutex for iternal synchronization. */
+ private final Object mux = new Object();
+
/** {@inheritDoc} */
@Override public void start(HadoopContext ctx) throws IgniteCheckedException {
super.start(ctx);
@@ -141,17 +144,23 @@ public class HadoopShuffle extends HadoopComponent {
HadoopShuffleJob<UUID> res = jobs.get(jobId);
if (res == null) {
- res = newJob(jobId);
+ synchronized (mux) {
+ res = jobs.get(jobId);
+
+ if (res == null) {
+ res = newJob(jobId);
- HadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
+ HadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
- if (old != null) {
- res.close();
+ if (old != null) {
+ res.close();
- res = old;
+ res = old;
+ }
+ else if (res.reducersInitialized())
+ startSending(res);
+ }
}
- else if (res.reducersInitialized())
- startSending(res);
}
return res;
http://git-wip-us.apache.org/repos/asf/ignite/blob/04cff9b5/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 9392b2c..aca5fdf 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -78,7 +78,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
private final boolean needPartitioner;
/** Collection of task contexts for each reduce task. */
- private final Map<Integer, HadoopTaskContext> reducersCtx = new HashMap<>();
+ private final Map<Integer, LocalTaskContextProxy> reducersCtx = new HashMap<>();
/** Reducers addresses. */
private T[] reduceAddrs;
@@ -139,7 +139,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
for (int rdc : locReducers) {
HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
- reducersCtx.put(rdc, job.getTaskContext(taskInfo));
+ reducersCtx.put(rdc, new LocalTaskContextProxy(taskInfo));
}
}
@@ -237,7 +237,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
assert msg.buffer() != null;
assert msg.offset() > 0;
- HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer());
+ HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer()).get();
HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
@@ -623,4 +623,51 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
}
}
+
+ /**
+ * Local task context proxy with delayed initialization.
+ */
+ private class LocalTaskContextProxy {
+ /** Mutex for synchronization. */
+ private final Object mux = new Object();
+
+ /** Task info. */
+ private final HadoopTaskInfo taskInfo;
+
+ /** Task context. */
+ private volatile HadoopTaskContext ctx;
+
+ /**
+ * Constructor.
+ *
+ * @param taskInfo Task info.
+ */
+ public LocalTaskContextProxy(HadoopTaskInfo taskInfo) {
+ this.taskInfo = taskInfo;
+ }
+
+ /**
+ * Get task context.
+ *
+ * @return Task context.
+ * @throws IgniteCheckedException If failed.
+ */
+ public HadoopTaskContext get() throws IgniteCheckedException {
+ HadoopTaskContext ctx0 = ctx;
+
+ if (ctx0 == null) {
+ synchronized (mux) {
+ ctx0 = ctx;
+
+ if (ctx0 == null) {
+ ctx0 = job.getTaskContext(taskInfo);
+
+ ctx = ctx0;
+ }
+ }
+ }
+
+ return ctx0;
+ }
+ }
}
\ No newline at end of file
[18/50] [abbrv] ignite git commit: IGNITE-4295: Added specialized
methods for memory copying,
depending on their nature (heap/offheap). Heap->heap copying is now performed
using System.arraycopy when possible. Also it is possible to fallback to
byte-by-b
Posted by sb...@apache.org.
IGNITE-4295: Added specialized methods for memory copying, depending on their nature (heap/offheap). Heap->heap copying is now performed using System.arraycopy when possible. Also it is possible to fallback to byte-by-byte copying until certain threshold is reached (see IgniteSystemProperties.IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD); it could be useful on some JDK implementations with inefficient Unsafe copy routines. E.g. on OpenJDK 8 for PowerPC. This closes #1282.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3c384212
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3c384212
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3c384212
Branch: refs/heads/master
Commit: 3c384212603dd31b933205d1c127dc9d0f1785e1
Parents: 2488f34
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 5 15:42:24 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:45:31 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 8 +++
.../internal/binary/BinaryPrimitives.java | 6 +-
.../binary/streams/BinaryHeapInputStream.java | 4 +-
.../binary/streams/BinaryHeapOutputStream.java | 2 +-
.../streams/BinaryMemoryAllocatorChunk.java | 3 +-
.../streams/BinaryOffheapInputStream.java | 4 +-
.../streams/BinaryOffheapOutputStream.java | 4 +-
.../memory/PlatformInputStreamImpl.java | 4 +-
.../memory/PlatformOutputStreamImpl.java | 2 +-
.../ignite/internal/util/GridHandleTable.java | 10 ++--
.../apache/ignite/internal/util/GridUnsafe.java | 60 ++++++++++++++++++--
.../ignite/internal/util/IgniteUtils.java | 32 +----------
.../internal/util/io/GridUnsafeDataInput.java | 12 ++--
.../internal/util/io/GridUnsafeDataOutput.java | 12 ++--
.../util/offheap/unsafe/GridUnsafeMemory.java | 10 ++--
.../binary/BinaryFieldsOffheapSelfTest.java | 2 +-
.../BinaryFooterOffsetsOffheapSelfTest.java | 2 +-
.../binary/BinaryMarshallerSelfTest.java | 2 +-
...naryObjectBuilderDefaultMappersSelfTest.java | 2 +-
.../hadoop/shuffle/HadoopShuffleJob.java | 2 +-
.../hadoop/shuffle/HadoopShuffleMessage.java | 2 +-
.../shuffle/streams/HadoopDataOutStream.java | 2 +-
.../HadoopConcurrentHashMultimapSelftest.java | 2 +-
.../collections/HadoopSkipListSelfTest.java | 2 +-
24 files changed, 108 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 16fc759..d0c0d5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -494,6 +494,14 @@ public final class IgniteSystemProperties {
public static final String IGNITE_MEMORY_UNALIGNED_ACCESS = "IGNITE_MEMORY_UNALIGNED_ACCESS";
/**
+ * When unsafe memory copy if performed below this threshold, Ignite will do it on per-byte basis instead of
+ * calling to Unsafe.copyMemory().
+ * <p>
+ * Defaults to 0, meaning that threshold is disabled.
+ */
+ public static final String IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD = "IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD";
+
+ /**
* Enforces singleton.
*/
private IgniteSystemProperties() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
index 86f5040..556282d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryPrimitives.java
@@ -60,7 +60,7 @@ public abstract class BinaryPrimitives {
public static byte[] readByteArray(byte[] arr, int off, int len) {
byte[] arr0 = new byte[len];
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF + off, arr0, GridUnsafe.BYTE_ARR_OFF, len);
+ System.arraycopy(arr, off, arr0, 0, len);
return arr0;
}
@@ -73,7 +73,7 @@ public abstract class BinaryPrimitives {
public static byte[] readByteArray(long ptr, int off, int len) {
byte[] arr0 = new byte[len];
- GridUnsafe.copyMemory(null, ptr + off, arr0, GridUnsafe.BYTE_ARR_OFF, len);
+ GridUnsafe.copyOffheapHeap(ptr + off, arr0, GridUnsafe.BYTE_ARR_OFF, len);
return arr0;
}
@@ -215,7 +215,7 @@ public abstract class BinaryPrimitives {
public static char[] readCharArray(long ptr, int off, int len) {
char[] arr0 = new char[len];
- GridUnsafe.copyMemory(null, ptr + off, arr0, GridUnsafe.CHAR_ARR_OFF, len << 1);
+ GridUnsafe.copyOffheapHeap(ptr + off, arr0, GridUnsafe.CHAR_ARR_OFF, len << 1);
if (BIG_ENDIAN) {
for (int i = 0; i < len; i++)
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
index b5edc02..322894e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapInputStream.java
@@ -78,7 +78,7 @@ public final class BinaryHeapInputStream extends BinaryAbstractInputStream {
if (data.length < len) {
byte[] data0 = new byte[len];
- GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, data0, GridUnsafe.BYTE_ARR_OFF, data.length);
+ System.arraycopy(data, 0, data0, 0, data.length);
data = data0;
}
@@ -105,7 +105,7 @@ public final class BinaryHeapInputStream extends BinaryAbstractInputStream {
@Override public byte[] arrayCopy() {
byte[] res = new byte[len];
- GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, res, GridUnsafe.BYTE_ARR_OFF, res.length);
+ System.arraycopy(data, 0, res, 0, len);
return res;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
index f06c980..17bcdf6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryHeapOutputStream.java
@@ -75,7 +75,7 @@ public final class BinaryHeapOutputStream extends BinaryAbstractOutputStream {
@Override public byte[] arrayCopy() {
byte[] res = new byte[pos];
- GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, res, GridUnsafe.BYTE_ARR_OFF, pos);
+ System.arraycopy(data, 0, res, 0, pos);
return res;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
index f9db7da..09f8c3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryMemoryAllocatorChunk.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.binary.streams;
-import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.typedef.internal.U;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHAL_BUFFERS_RECHECK;
@@ -72,7 +71,7 @@ public class BinaryMemoryAllocatorChunk {
if (this.data == data)
this.data = newData;
- GridUnsafe.copyMemory(data, GridUnsafe.BYTE_ARR_OFF, newData, GridUnsafe.BYTE_ARR_OFF, data.length);
+ System.arraycopy(data, 0, newData, 0, data.length);
return newData;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
index 9dc92c6..6f505b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapInputStream.java
@@ -79,7 +79,7 @@ public class BinaryOffheapInputStream extends BinaryAbstractInputStream {
@Override public byte[] arrayCopy() {
byte[] res = new byte[len];
- GridUnsafe.copyMemory(null, ptr, res, GridUnsafe.BYTE_ARR_OFF, res.length);
+ GridUnsafe.copyOffheapHeap(ptr, res, GridUnsafe.BYTE_ARR_OFF, res.length);
return res;
}
@@ -96,7 +96,7 @@ public class BinaryOffheapInputStream extends BinaryAbstractInputStream {
/** {@inheritDoc} */
@Override protected void copyAndShift(Object target, long off, int len) {
- GridUnsafe.copyMemory(null, ptr + pos, target, off, len);
+ GridUnsafe.copyOffheapHeap(ptr + pos, target, off, len);
shift(len);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
index be9f7d3..d16e575 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOffheapOutputStream.java
@@ -77,7 +77,7 @@ public class BinaryOffheapOutputStream extends BinaryAbstractOutputStream {
@Override public byte[] arrayCopy() {
byte[] res = new byte[pos];
- GridUnsafe.copyMemory(null, ptr, res, GridUnsafe.BYTE_ARR_OFF, pos);
+ GridUnsafe.copyOffheapHeap(ptr, res, GridUnsafe.BYTE_ARR_OFF, pos);
return res;
}
@@ -101,7 +101,7 @@ public class BinaryOffheapOutputStream extends BinaryAbstractOutputStream {
/** {@inheritDoc} */
@Override protected void copyAndShift(Object src, long offset, int len) {
- GridUnsafe.copyMemory(src, offset, null, ptr + pos, len);
+ GridUnsafe.copyHeapOffheap(src, offset, ptr + pos, len);
shift(len);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
index 859de39..f5b2c3a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformInputStreamImpl.java
@@ -286,7 +286,7 @@ public class PlatformInputStreamImpl implements PlatformInputStream {
if (dataCopy == null) {
dataCopy = new byte[len];
- GridUnsafe.copyMemory(null, data, dataCopy, GridUnsafe.BYTE_ARR_OFF, dataCopy.length);
+ GridUnsafe.copyOffheapHeap(data, dataCopy, GridUnsafe.BYTE_ARR_OFF, dataCopy.length);
}
return dataCopy;
@@ -334,7 +334,7 @@ public class PlatformInputStreamImpl implements PlatformInputStream {
private void copyAndShift(Object target, long off, int cnt) {
ensureEnoughData(cnt);
- GridUnsafe.copyMemory(null, data + pos, target, off, cnt);
+ GridUnsafe.copyOffheapHeap(data + pos, target, off, cnt);
shift(cnt);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
index 1ece10b..334afb8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java
@@ -334,7 +334,7 @@ public class PlatformOutputStreamImpl implements PlatformOutputStream {
private void copyAndShift(Object src, long off, int len) {
ensureCapacity(pos + len);
- GridUnsafe.copyMemory(src, off, null, data + pos, len);
+ GridUnsafe.copyHeapOffheap(src, off, data + pos, len);
shift(len);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
index 41a4da9..aace1ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridHandleTable.java
@@ -19,8 +19,6 @@ package org.apache.ignite.internal.util;
import java.util.Arrays;
-import static org.apache.ignite.internal.util.GridUnsafe.INT_ARR_OFF;
-
/**
* Lightweight identity hash table which maps objects to integer handles,
* assigned in ascending order.
@@ -109,8 +107,8 @@ public class GridHandleTable {
* Resets table to its initial (empty) state.
*/
public void clear() {
- GridUnsafe.copyMemory(spineEmpty, INT_ARR_OFF, spine, INT_ARR_OFF, spineEmpty.length << 2);
- GridUnsafe.copyMemory(nextEmpty, INT_ARR_OFF, next, INT_ARR_OFF, nextEmpty.length << 2);
+ System.arraycopy(spineEmpty, 0, spine, 0, spineEmpty.length);
+ System.arraycopy(nextEmpty, 0, next, 0, nextEmpty.length);
Arrays.fill(objs, null);
@@ -151,7 +149,7 @@ public class GridHandleTable {
Arrays.fill(spineEmpty, -1);
- GridUnsafe.copyMemory(spineEmpty, INT_ARR_OFF, spine, INT_ARR_OFF, spineEmpty.length << 2);
+ System.arraycopy(spineEmpty, 0, spine, 0, spineEmpty.length);
for (int i = 0; i < this.size; i++) {
Object obj = objs[i];
@@ -169,7 +167,7 @@ public class GridHandleTable {
int newLen = (next.length << 1) + 1;
int[] newNext = new int[newLen];
- GridUnsafe.copyMemory(next, INT_ARR_OFF, newNext, INT_ARR_OFF, size << 2);
+ System.arraycopy(next, 0, newNext, 0, size);
next = newNext;
nextEmpty = new int[newLen];
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
index 35b793a..6e9efdb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridUnsafe.java
@@ -51,6 +51,10 @@ public abstract class GridUnsafe {
/** Unaligned flag. */
private static final boolean UNALIGNED = unaligned();
+ /** Per-byte copy threshold. */
+ private static final long PER_BYTE_THRESHOLD =
+ IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_MEMORY_PER_BYTE_COPY_THRESHOLD, 0L);
+
/** Big endian. */
public static final boolean BIG_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN;
@@ -1027,14 +1031,53 @@ public abstract class GridUnsafe {
}
/**
- * Copies memory.
+ * Copy memory between offheap locations.
+ *
+ * @param srcAddr Source address.
+ * @param dstAddr Destination address.
+ * @param len Length.
+ */
+ public static void copyOffheapOffheap(long srcAddr, long dstAddr, long len) {
+ if (len <= PER_BYTE_THRESHOLD) {
+ for (int i = 0; i < len; i++)
+ UNSAFE.putByte(dstAddr + i, UNSAFE.getByte(srcAddr + i));
+ }
+ else
+ UNSAFE.copyMemory(srcAddr, dstAddr, len);
+ }
+
+ /**
+ * Copy memory from offheap to heap.
*
- * @param src Source.
- * @param dst Dst.
+ * @param srcAddr Source address.
+ * @param dstBase Destination base.
+ * @param dstOff Destination offset.
* @param len Length.
*/
- public static void copyMemory(long src, long dst, long len) {
- UNSAFE.copyMemory(src, dst, len);
+ public static void copyOffheapHeap(long srcAddr, Object dstBase, long dstOff, long len) {
+ if (len <= PER_BYTE_THRESHOLD) {
+ for (int i = 0; i < len; i++)
+ UNSAFE.putByte(dstBase, dstOff + i, UNSAFE.getByte(srcAddr + i));
+ }
+ else
+ UNSAFE.copyMemory(null, srcAddr, dstBase, dstOff, len);
+ }
+
+ /**
+ * Copy memory from heap to offheap.
+ *
+ * @param srcBase Source base.
+ * @param srcOff Source offset.
+ * @param dstAddr Destination address.
+ * @param len Length.
+ */
+ public static void copyHeapOffheap(Object srcBase, long srcOff, long dstAddr, long len) {
+ if (len <= PER_BYTE_THRESHOLD) {
+ for (int i = 0; i < len; i++)
+ UNSAFE.putByte(dstAddr + i, UNSAFE.getByte(srcBase, srcOff + i));
+ }
+ else
+ UNSAFE.copyMemory(srcBase, srcOff, null, dstAddr, len);
}
/**
@@ -1047,7 +1090,12 @@ public abstract class GridUnsafe {
* @param len Length.
*/
public static void copyMemory(Object srcBase, long srcOff, Object dstBase, long dstOff, long len) {
- UNSAFE.copyMemory(srcBase, srcOff, dstBase, dstOff, len);
+ if (len <= PER_BYTE_THRESHOLD && srcBase != null && dstBase != null) {
+ for (int i = 0; i < len; i++)
+ UNSAFE.putByte(dstBase, dstOff + i, UNSAFE.getByte(srcBase, srcOff + i));
+ }
+ else
+ UNSAFE.copyMemory(srcBase, srcOff, dstBase, dstOff, len);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3fa3f7b..e1937bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -265,9 +265,6 @@ import static org.apache.ignite.internal.util.GridUnsafe.staticFieldOffset;
*/
@SuppressWarnings({"UnusedReturnValue", "UnnecessaryFullyQualifiedName", "RedundantStringConstructorCall"})
public abstract class IgniteUtils {
- /** {@code True} if {@code unsafe} should be used for array copy. */
- private static final boolean UNSAFE_BYTE_ARR_CP = unsafeByteArrayCopyAvailable();
-
/** Sun-specific JDK constructor factory for objects that don't have empty constructor. */
private static final Method CTOR_FACTORY;
@@ -8482,28 +8479,6 @@ public abstract class IgniteUtils {
}
/**
- * As long as array copying uses JVM-private API, which is not guaranteed
- * to be available on all JVM, this method should be called to ensure
- * logic could work properly.
- *
- * @return {@code True} if unsafe copying can work on the current JVM or
- * {@code false} if it can't.
- */
- @SuppressWarnings("TypeParameterExtendsFinalClass")
- private static boolean unsafeByteArrayCopyAvailable() {
- try {
- Class<? extends Unsafe> unsafeCls = Unsafe.class;
-
- unsafeCls.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class);
-
- return true;
- }
- catch (Exception ignored) {
- return false;
- }
- }
-
- /**
* @param src Buffer to copy from (length included).
* @param off Offset in source buffer.
* @param resBuf Result buffer.
@@ -8514,10 +8489,7 @@ public abstract class IgniteUtils {
public static int arrayCopy(byte[] src, int off, byte[] resBuf, int resOff, int len) {
assert resBuf.length >= resOff + len;
- if (UNSAFE_BYTE_ARR_CP)
- GridUnsafe.copyMemory(src, GridUnsafe.BYTE_ARR_OFF + off, resBuf, GridUnsafe.BYTE_ARR_OFF + resOff, len);
- else
- System.arraycopy(src, off, resBuf, resOff, len);
+ System.arraycopy(src, off, resBuf, resOff, len);
return resOff + len;
}
@@ -9284,7 +9256,7 @@ public abstract class IgniteUtils {
public static byte[] copyMemory(long ptr, int size) {
byte[] res = new byte[size];
- GridUnsafe.copyMemory(null, ptr, res, GridUnsafe.BYTE_ARR_OFF, size);
+ GridUnsafe.copyOffheapHeap(ptr, res, GridUnsafe.BYTE_ARR_OFF, size);
return res;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
index 2f57e59..759b95e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataInput.java
@@ -135,7 +135,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
if (maxOff < halfSize) {
byte[] newInBuf = new byte[halfSize]; // Shrink.
- GridUnsafe.copyMemory(inBuf, BYTE_ARR_OFF, newInBuf, BYTE_ARR_OFF, off);
+ System.arraycopy(inBuf, 0, newInBuf, 0, off);
buf = inBuf = newInBuf;
}
@@ -191,7 +191,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
byte[] arr = new byte[arrSize];
- GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(arrSize), arr, BYTE_ARR_OFF, arrSize);
+ System.arraycopy(buf, offset(arrSize), arr, 0, arrSize);
return arr;
}
@@ -364,14 +364,14 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
fromStream(len);
- GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(len), b, BYTE_ARR_OFF, len);
+ System.arraycopy(buf, offset(len), b, 0, len);
}
/** {@inheritDoc} */
@Override public void readFully(byte[] b, int off, int len) throws IOException {
fromStream(len);
- GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(len), b, BYTE_ARR_OFF + off, len);
+ System.arraycopy(buf, offset(len), b, off, len);
}
/** {@inheritDoc} */
@@ -488,7 +488,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
else {
int toRead = Math.min(len, max - this.off);
- GridUnsafe.copyMemory(buf, BYTE_ARR_OFF + offset(toRead), b, BYTE_ARR_OFF + off, toRead);
+ System.arraycopy(buf, offset(toRead), b, off, toRead);
return toRead;
}
@@ -552,7 +552,7 @@ public class GridUnsafeDataInput extends InputStream implements GridDataInput {
else {
// shift and refill buffer manually
if (avail > 0)
- GridUnsafe.copyMemory(utfBuf, BYTE_ARR_OFF + pos, utfBuf, BYTE_ARR_OFF, avail);
+ System.arraycopy(utfBuf, pos, utfBuf, 0, avail);
pos = 0;
end = (int)Math.min(MAX_BLOCK_SIZE, utfLen);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
index c0fe0d3..c45b8fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
@@ -97,7 +97,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
@Override public byte[] array() {
byte[] bytes0 = new byte[off];
- GridUnsafe.copyMemory(bytes, BYTE_ARR_OFF, bytes0, BYTE_ARR_OFF, off);
+ System.arraycopy(bytes, 0, bytes0, 0, off);
return bytes0;
}
@@ -130,7 +130,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
if (size > bytes.length) {
byte[] newBytes = new byte[size << 1]; // Grow.
- GridUnsafe.copyMemory(bytes, BYTE_ARR_OFF, newBytes, BYTE_ARR_OFF, off);
+ System.arraycopy(bytes, 0, newBytes, 0, off);
bytes = newBytes;
}
@@ -140,7 +140,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
if (maxOff < halfSize) {
byte[] newBytes = new byte[halfSize]; // Shrink.
- GridUnsafe.copyMemory(bytes, BYTE_ARR_OFF, newBytes, BYTE_ARR_OFF, off);
+ System.arraycopy(bytes, 0, newBytes, 0, off);
bytes = newBytes;
}
@@ -165,7 +165,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
@Override public void write(byte[] b) throws IOException {
requestFreeSize(b.length);
- GridUnsafe.copyMemory(b, BYTE_ARR_OFF, bytes, BYTE_ARR_OFF + off, b.length);
+ System.arraycopy(b, 0, bytes, off, b.length);
onWrite(b.length);
}
@@ -174,7 +174,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
@Override public void write(byte[] b, int off, int len) throws IOException {
requestFreeSize(len);
- GridUnsafe.copyMemory(b, BYTE_ARR_OFF + off, bytes, BYTE_ARR_OFF + this.off, len);
+ System.arraycopy(b, off, bytes, this.off, len);
onWrite(len);
}
@@ -293,7 +293,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
requestFreeSize(arr.length);
- GridUnsafe.copyMemory(arr, BYTE_ARR_OFF, bytes, BYTE_ARR_OFF + off, arr.length);
+ System.arraycopy(arr, 0, bytes, off, arr.length);
onWrite(arr.length);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
index 718e1a6..41cb586 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridUnsafeMemory.java
@@ -607,7 +607,7 @@ public class GridUnsafeMemory {
* @return The same array as passed in one.
*/
public byte[] readBytes(long ptr, byte[] arr) {
- GridUnsafe.copyMemory(null, ptr, arr, GridUnsafe.BYTE_ARR_OFF, arr.length);
+ GridUnsafe.copyOffheapHeap(ptr, arr, GridUnsafe.BYTE_ARR_OFF, arr.length);
return arr;
}
@@ -620,7 +620,7 @@ public class GridUnsafeMemory {
* @return The same array as passed in one.
*/
public byte[] readBytes(long ptr, byte[] arr, int off, int len) {
- GridUnsafe.copyMemory(null, ptr, arr, GridUnsafe.BYTE_ARR_OFF + off, len);
+ GridUnsafe.copyOffheapHeap(ptr, arr, GridUnsafe.BYTE_ARR_OFF + off, len);
return arr;
}
@@ -632,7 +632,7 @@ public class GridUnsafeMemory {
* @param arr Array.
*/
public void writeBytes(long ptr, byte[] arr) {
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+ GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
}
/**
@@ -644,7 +644,7 @@ public class GridUnsafeMemory {
* @param len Length.
*/
public void writeBytes(long ptr, byte[] arr, int off, int len) {
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF + off, null, ptr, len);
+ GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF + off, ptr, len);
}
/**
@@ -655,7 +655,7 @@ public class GridUnsafeMemory {
* @param len Length in bytes.
*/
public void copyMemory(long srcPtr, long destPtr, long len) {
- GridUnsafe.copyMemory(srcPtr, destPtr, len);
+ GridUnsafe.copyOffheapOffheap(srcPtr, destPtr, len);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
index 1546252..ca4bdd2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldsOffheapSelfTest.java
@@ -46,7 +46,7 @@ public class BinaryFieldsOffheapSelfTest extends BinaryFieldsAbstractSelfTest {
ptrs.add(ptr);
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+ GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
return new BinaryObjectOffheapImpl(binaryContext(marsh), ptr, 0, arr.length);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
index 796c027..052e652 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFooterOffsetsOffheapSelfTest.java
@@ -46,7 +46,7 @@ public class BinaryFooterOffsetsOffheapSelfTest extends BinaryFooterOffsetsAbstr
ptrs.add(ptr);
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+ GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
return new BinaryObjectOffheapImpl(ctx, ptr, 0, arr.length);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 39a4d32..70df1e6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -3346,7 +3346,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest {
long ptr = GridUnsafe.allocateMemory(arr.length);
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr, arr.length);
+ GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr, arr.length);
return ptr;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
index 4fc5c6c..b623c62 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryObjectBuilderDefaultMappersSelfTest.java
@@ -791,7 +791,7 @@ public class BinaryObjectBuilderDefaultMappersSelfTest extends GridCommonAbstrac
else
GridUnsafe.putInt(ptr0, len);
- GridUnsafe.copyMemory(arr, GridUnsafe.BYTE_ARR_OFF, null, ptr0 + 4, arr.length);
+ GridUnsafe.copyHeapOffheap(arr, GridUnsafe.BYTE_ARR_OFF, ptr0 + 4, arr.length);
BinaryObject offheapObj = (BinaryObject)
((CacheObjectBinaryProcessorImpl)(grid(0)).context().cacheObjects()).unmarshal(ptr, false);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index e5af8f1..025c4da 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -311,7 +311,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/** */
@Override public void copyTo(long ptr) {
- GridUnsafe.copyMemory(buf, GridUnsafe.BYTE_ARR_OFF + off, null, ptr, size);
+ GridUnsafe.copyHeapOffheap(buf, GridUnsafe.BYTE_ARR_OFF + off, ptr, size);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
index 69dfe64..71a314b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -169,7 +169,7 @@ public class HadoopShuffleMessage implements HadoopMessage {
off += 4;
- GridUnsafe.copyMemory(null, ptr, buf, GridUnsafe.BYTE_ARR_OFF + off, size);
+ GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF + off, size);
off += size;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
index f7b1a73..bb5acba 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/streams/HadoopDataOutStream.java
@@ -67,7 +67,7 @@ public class HadoopDataOutStream extends OutputStream implements DataOutput {
/** {@inheritDoc} */
@Override public void write(byte[] b, int off, int len) {
- GridUnsafe.copyMemory(b, GridUnsafe.BYTE_ARR_OFF + off, null, move(len), len);
+ GridUnsafe.copyHeapOffheap(b, GridUnsafe.BYTE_ARR_OFF + off, move(len), len);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
index 019b172..7862d6e 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopConcurrentHashMultimapSelftest.java
@@ -162,7 +162,7 @@ public class HadoopConcurrentHashMultimapSelftest extends HadoopAbstractMapTest
private void read(long ptr, int size, Writable w) {
assert size == 4 : size;
- GridUnsafe.copyMemory(null, ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
+ GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
dataInput.bytes(buf, size);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c384212/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
index d04beca..111ea78 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
@@ -202,7 +202,7 @@ public class HadoopSkipListSelfTest extends HadoopAbstractMapTest {
private void read(long ptr, int size, Writable w) {
assert size == 4 : size;
- GridUnsafe.copyMemory(null, ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
+ GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF, size);
dataInput.bytes(buf, size);
[34/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index 63c9845..66f9176 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -20,9 +20,11 @@ package org.apache.ignite.internal.util.nio;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.SelectionKey;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -37,17 +39,14 @@ import org.jsr166.ConcurrentLinkedDeque8;
*/
class GridSelectorNioSessionImpl extends GridNioSessionImpl {
/** Pending write requests. */
- private final ConcurrentLinkedDeque8<GridNioFuture<?>> queue = new ConcurrentLinkedDeque8<>();
+ private final ConcurrentLinkedDeque8<SessionWriteRequest> queue = new ConcurrentLinkedDeque8<>();
/** Selection key associated with this session. */
@GridToStringExclude
private SelectionKey key;
- /** Worker index for server */
- private final int selectorIdx;
-
- /** Size counter. */
- private final AtomicInteger queueSize = new AtomicInteger();
+ /** Current worker thread. */
+ private volatile GridNioWorker worker;
/** Semaphore. */
@GridToStringExclude
@@ -59,17 +58,29 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
/** Read buffer. */
private ByteBuffer readBuf;
- /** Recovery data. */
- private GridNioRecoveryDescriptor recovery;
+ /** Incoming recovery data. */
+ private GridNioRecoveryDescriptor inRecovery;
+
+ /** Outgoing recovery data. */
+ private GridNioRecoveryDescriptor outRecovery;
/** Logger. */
private final IgniteLogger log;
+ /** */
+ private List<GridNioServer.SessionChangeRequest> pendingStateChanges;
+
+ /** */
+ final AtomicBoolean procWrite = new AtomicBoolean();
+
+ /** */
+ private Object sysMsg;
+
/**
* Creates session instance.
*
* @param log Logger.
- * @param selectorIdx Selector index for this session.
+ * @param worker NIO worker thread.
* @param filterChain Filter chain that will handle requests.
* @param locAddr Local address.
* @param rmtAddr Remote address.
@@ -80,7 +91,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
*/
GridSelectorNioSessionImpl(
IgniteLogger log,
- int selectorIdx,
+ GridNioWorker worker,
GridNioFilterChain filterChain,
InetSocketAddress locAddr,
InetSocketAddress rmtAddr,
@@ -91,7 +102,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
) {
super(filterChain, locAddr, rmtAddr, accepted);
- assert selectorIdx >= 0;
+ assert worker != null;
assert sndQueueLimit >= 0;
assert locAddr != null : "GridSelectorNioSessionImpl should have local socket address.";
@@ -101,7 +112,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
this.log = log;
- this.selectorIdx = selectorIdx;
+ this.worker = worker;
sem = sndQueueLimit > 0 ? new Semaphore(sndQueueLimit) : null;
@@ -119,12 +130,19 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
}
/**
+ * @return Worker.
+ */
+ GridNioWorker worker() {
+ return worker;
+ }
+
+ /**
* Sets selection key for this session.
*
* @param key Selection key.
*/
void key(SelectionKey key) {
- assert this.key == null;
+ assert key != null;
this.key = key;
}
@@ -151,10 +169,88 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
}
/**
- * @return Selector index.
+ * @param from Current session worker.
+ * @param fut Move future.
+ * @return {@code True} if session move was scheduled.
+ */
+ boolean offerMove(GridNioWorker from, GridNioServer.SessionChangeRequest fut) {
+ synchronized (this) {
+ if (log.isDebugEnabled())
+ log.debug("Offered move [ses=" + this + ", fut=" + fut + ']');
+
+ GridNioWorker worker0 = worker;
+
+ if (worker0 != from)
+ return false;
+
+ worker.offer(fut);
+ }
+
+ return true;
+ }
+
+ /**
+ * @param fut Future.
+ */
+ void offerStateChange(GridNioServer.SessionChangeRequest fut) {
+ synchronized (this) {
+ if (log.isDebugEnabled())
+ log.debug("Offered move [ses=" + this + ", fut=" + fut + ']');
+
+ GridNioWorker worker0 = worker;
+
+ if (worker0 == null) {
+ if (pendingStateChanges == null)
+ pendingStateChanges = new ArrayList<>();
+
+ pendingStateChanges.add(fut);
+ }
+ else
+ worker0.offer(fut);
+ }
+ }
+
+ /**
+ * @param moveFrom Current session worker.
*/
- int selectorIndex() {
- return selectorIdx;
+ void startMoveSession(GridNioWorker moveFrom) {
+ synchronized (this) {
+ assert this.worker == moveFrom;
+
+ if (log.isDebugEnabled())
+ log.debug("Started moving [ses=" + this + ", from=" + moveFrom + ']');
+
+ List<GridNioServer.SessionChangeRequest> sesReqs = moveFrom.clearSessionRequests(this);
+
+ worker = null;
+
+ if (sesReqs != null) {
+ if (pendingStateChanges == null)
+ pendingStateChanges = new ArrayList<>();
+
+ pendingStateChanges.addAll(sesReqs);
+ }
+ }
+ }
+
+ /**
+ * @param moveTo New session worker.
+ */
+ void finishMoveSession(GridNioWorker moveTo) {
+ synchronized (this) {
+ assert worker == null;
+
+ if (log.isDebugEnabled())
+ log.debug("Finishing moving [ses=" + this + ", to=" + moveTo + ']');
+
+ worker = moveTo;
+
+ if (pendingStateChanges != null) {
+ moveTo.offer(pendingStateChanges);
+
+ pendingStateChanges = null;
+ }
+ }
}
/**
@@ -163,14 +259,14 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
* @param writeFut Write request.
* @return Updated size of the queue.
*/
- int offerSystemFuture(GridNioFuture<?> writeFut) {
+ int offerSystemFuture(SessionWriteRequest writeFut) {
writeFut.messageThread(true);
boolean res = queue.offerFirst(writeFut);
assert res : "Future was not added to queue";
- return queueSize.incrementAndGet();
+ return queue.sizex();
}
/**
@@ -183,7 +279,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
* @param writeFut Write request to add.
* @return Updated size of the queue.
*/
- int offerFuture(GridNioFuture<?> writeFut) {
+ int offerFuture(SessionWriteRequest writeFut) {
boolean msgThread = GridNioBackPressureControl.threadProcessingMessage();
if (sem != null && !msgThread)
@@ -195,47 +291,41 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
assert res : "Future was not added to queue";
- return queueSize.incrementAndGet();
+ return queue.sizex();
}
/**
* @param futs Futures to resend.
*/
- void resend(Collection<GridNioFuture<?>> futs) {
+ void resend(Collection<SessionWriteRequest> futs) {
assert queue.isEmpty() : queue.size();
boolean add = queue.addAll(futs);
assert add;
-
- boolean set = queueSize.compareAndSet(0, futs.size());
-
- assert set;
}
/**
* @return Message that is in the head of the queue, {@code null} if queue is empty.
*/
- @Nullable GridNioFuture<?> pollFuture() {
- GridNioFuture<?> last = queue.poll();
+ @Nullable SessionWriteRequest pollFuture() {
+ SessionWriteRequest last = queue.poll();
if (last != null) {
- queueSize.decrementAndGet();
-
if (sem != null && !last.messageThread())
sem.release();
- if (recovery != null) {
- if (!recovery.add(last)) {
+ if (outRecovery != null) {
+ if (!outRecovery.add(last)) {
LT.warn(log, "Unacknowledged messages queue size overflow, will attempt to reconnect " +
"[remoteAddr=" + remoteAddress() +
- ", queueLimit=" + recovery.queueLimit() + ']');
+ ", queueLimit=" + outRecovery.queueLimit() + ']');
if (log.isDebugEnabled())
log.debug("Unacknowledged messages queue size overflow, will attempt to reconnect " +
"[remoteAddr=" + remoteAddress() +
- ", queueSize=" + recovery.messagesFutures().size() +
- ", queueLimit=" + recovery.queueLimit() + ']');
+ ", queueSize=" + outRecovery.messagesRequests().size() +
+ ", queueLimit=" + outRecovery.queueLimit() + ']');
close();
}
@@ -249,7 +339,7 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
* @param fut Future.
* @return {@code True} if future was removed from queue.
*/
- boolean removeFuture(GridNioFuture<?> fut) {
+ boolean removeFuture(SessionWriteRequest fut) {
assert closed();
return queue.removeLastOccurrence(fut);
@@ -261,35 +351,49 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
* @return Number of write requests.
*/
int writeQueueSize() {
- return queueSize.get();
+ return queue.sizex();
}
/**
* @return Write requests.
*/
- Collection<GridNioFuture<?>> writeQueue() {
+ Collection<SessionWriteRequest> writeQueue() {
return queue;
}
/** {@inheritDoc} */
- @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
assert recoveryDesc != null;
- recovery = recoveryDesc;
+ outRecovery = recoveryDesc;
}
/** {@inheritDoc} */
- @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
- return recovery;
+ @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
+ return outRecovery;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ assert recoveryDesc != null;
+
+ inRecovery = recoveryDesc;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
+ return inRecovery;
}
/** {@inheritDoc} */
@Override public <T> T addMeta(int key, @Nullable T val) {
- if (val instanceof GridNioRecoveryDescriptor) {
- recovery = (GridNioRecoveryDescriptor)val;
+ if (!accepted() && val instanceof GridNioRecoveryDescriptor) {
+ outRecovery = (GridNioRecoveryDescriptor)val;
+
+ if (!outRecovery.pairedConnections())
+ inRecovery = outRecovery;
- if (!accepted())
- recovery.connected();
+ outRecovery.onConnected();
return null;
}
@@ -313,6 +417,31 @@ class GridSelectorNioSessionImpl extends GridNioSessionImpl {
}
/** {@inheritDoc} */
+ @Override public void systemMessage(Object sysMsg) {
+ this.sysMsg = sysMsg;
+ }
+
+ /**
+ * @return {@code True} if have pending system message to send.
+ */
+ boolean hasSystemMessage() {
+ return sysMsg != null;
+ }
+
+ /**
+ * Gets and clears pending system message.
+ *
+ * @return Pending system message.
+ */
+ Object systemMessage() {
+ Object ret = sysMsg;
+
+ sysMsg = null;
+
+ return ret;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridSelectorNioSessionImpl.class, this, super.toString());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
index ebe86fb..d941bae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridShmemCommunicationClient.java
@@ -48,6 +48,7 @@ public class GridShmemCommunicationClient extends GridAbstractCommunicationClien
private final MessageFormatter formatter;
/**
+ * @param connIdx Connection index.
* @param metricsLsnr Metrics listener.
* @param port Shared memory IPC server port.
* @param connTimeout Connection timeout.
@@ -55,14 +56,16 @@ public class GridShmemCommunicationClient extends GridAbstractCommunicationClien
* @param formatter Message formatter.
* @throws IgniteCheckedException If failed.
*/
- public GridShmemCommunicationClient(GridNioMetricsListener metricsLsnr,
+ public GridShmemCommunicationClient(
+ int connIdx,
+ GridNioMetricsListener metricsLsnr,
int port,
long connTimeout,
IgniteLogger log,
MessageFormatter formatter)
throws IgniteCheckedException
{
- super(metricsLsnr);
+ super(connIdx, metricsLsnr);
assert metricsLsnr != null;
assert port > 0 && port < 0xffff;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
index 5fe521d..3397772 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridTcpNioCommunicationClient.java
@@ -45,11 +45,16 @@ public class GridTcpNioCommunicationClient extends GridAbstractCommunicationClie
private final IgniteLogger log;
/**
+ * @param connIdx Connection index.
* @param ses Session.
* @param log Logger.
*/
- public GridTcpNioCommunicationClient(GridNioSession ses, IgniteLogger log) {
- super(null);
+ public GridTcpNioCommunicationClient(
+ int connIdx,
+ GridNioSession ses,
+ IgniteLogger log
+ ) {
+ super(connIdx, null);
assert ses != null;
assert log != null;
@@ -104,40 +109,36 @@ public class GridTcpNioCommunicationClient extends GridAbstractCommunicationClie
}
/** {@inheritDoc} */
- @Override public boolean sendMessage(@Nullable UUID nodeId, Message msg, IgniteInClosure<IgniteException> closure)
+ @Override public boolean sendMessage(@Nullable UUID nodeId, Message msg, IgniteInClosure<IgniteException> c)
throws IgniteCheckedException {
- // Node ID is never provided in asynchronous send mode.
- assert nodeId == null;
+ try {
+ // Node ID is never provided in asynchronous send mode.
+ assert nodeId == null;
- if (closure != null)
- ses.addMeta(ACK_CLOSURE.ordinal(), closure);
+ if (c != null)
+ ses.addMeta(ACK_CLOSURE.ordinal(), c);
- GridNioFuture<?> fut = ses.send(msg);
+ ses.sendNoFuture(msg);
- if (fut.isDone()) {
- try {
- fut.get();
- }
- catch (IgniteCheckedException e) {
- if (closure != null)
- ses.removeMeta(ACK_CLOSURE.ordinal());
+ if (c != null)
+ ses.removeMeta(ACK_CLOSURE.ordinal());
+ }
+ catch (IgniteCheckedException e) {
+ if (c != null)
+ ses.removeMeta(ACK_CLOSURE.ordinal());
- if (log.isDebugEnabled())
- log.debug("Failed to send message [client=" + this + ", err=" + e + ']');
+ if (log.isDebugEnabled())
+ log.debug("Failed to send message [client=" + this + ", err=" + e + ']');
- if (e.getCause() instanceof IOException) {
- ses.close();
+ if (e.getCause() instanceof IOException) {
+ ses.close();
- return true;
- }
- else
- throw new IgniteCheckedException("Failed to send message [client=" + this + ']', e);
+ return true;
}
+ else
+ throw new IgniteCheckedException("Failed to send message [client=" + this + ']', e);
}
- if (closure != null)
- ses.removeMeta(ACK_CLOSURE.ordinal());
-
return false;
}
@@ -159,4 +160,4 @@ public class GridTcpNioCommunicationClient extends GridAbstractCommunicationClie
@Override public String toString() {
return S.toString(GridTcpNioCommunicationClient.class, this, super.toString());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java
new file mode 100644
index 0000000..508c791
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/SessionWriteRequest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.nio;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.lang.IgniteInClosure;
+
+/**
+ *
+ */
+public interface SessionWriteRequest {
+ /**
+ * Sets flag indicating that message send future was created in thread that was processing a message.
+ *
+ * @param msgThread {@code True} if future was created in thread that is processing message.
+ */
+ public void messageThread(boolean msgThread);
+
+ /**
+ * @return {@code True} if future was created in thread that was processing message.
+ */
+ public boolean messageThread();
+
+ /**
+ * @return {@code True} if skip recovery for this operation.
+ */
+ public boolean skipRecovery();
+
+ /**
+ * Sets ack closure which will be applied when ack received.
+ *
+ * @param c Ack closure.
+ */
+ public void ackClosure(IgniteInClosure<IgniteException> c);
+
+ /**
+ * The method will be called when ack received.
+ */
+ public void onAckReceived();
+
+ /**
+ * @return Ack closure.
+ */
+ public IgniteInClosure<IgniteException> ackClosure();
+
+ /**
+ * @return Session.
+ */
+ public GridNioSession session();
+
+ /**
+ * @param ses Session.
+ */
+ public void resetSession(GridNioSession ses);
+
+ /**
+ *
+ */
+ public void onError(Exception e);
+
+ /**
+ * @return Message.
+ */
+ public Object message();
+
+ /**
+ *
+ */
+ public void onMessageWritten();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
index d6f9d10..8ed7db0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslFilter.java
@@ -282,9 +282,13 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ @Override public GridNioFuture<?> onSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException {
if (directMode)
- return proceedSessionWrite(ses, msg);
+ return proceedSessionWrite(ses, msg, fut);
ByteBuffer input = checkMessage(ses, msg);
@@ -441,4 +445,4 @@ public class GridNioSslFilter extends GridNioFilterAdapter {
return (ByteBuffer)msg;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
index eb8dad4..269e8b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/GridNioSslHandler.java
@@ -437,7 +437,7 @@ class GridNioSslHandler extends ReentrantLock {
while (!deferredWriteQueue.isEmpty()) {
WriteRequest req = deferredWriteQueue.poll();
- req.future().onDone((GridNioFuture<Object>)parent.proceedSessionWrite(ses, req.buffer()));
+ req.future().onDone((GridNioFuture<Object>)parent.proceedSessionWrite(ses, req.buffer(), true));
}
}
@@ -482,7 +482,7 @@ class GridNioSslHandler extends ReentrantLock {
ByteBuffer cp = copy(outNetBuf);
- return parent.proceedSessionWrite(ses, cp);
+ return parent.proceedSessionWrite(ses, cp, true);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index b29d7cd..86aa7a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -598,4 +598,4 @@ public class GridToStringBuilder {
return cd;
}
-}
\ No newline at end of file
+}
[49/50] [abbrv] ignite git commit: GridTopic.TOPIC_HADOOP_MSG: new
topic should be added at the end for compatibility with previous releases.
Posted by sb...@apache.org.
GridTopic.TOPIC_HADOOP_MSG: new topic should be added at the end for compatibility with previous releases.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8372e69d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8372e69d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8372e69d
Branch: refs/heads/master
Commit: 8372e69dd01972a01dbd59689475f2d45c2b6d94
Parents: ce46c10
Author: sboikov <sb...@gridgain.com>
Authored: Thu Dec 22 10:30:05 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Dec 22 10:30:05 2016 +0300
----------------------------------------------------------------------
.../src/main/java/org/apache/ignite/internal/GridTopic.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8372e69d/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 24ddcd2..2962540 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -91,16 +91,16 @@ public enum GridTopic {
TOPIC_HADOOP,
/** */
- TOPIC_HADOOP_MSG,
-
- /** */
TOPIC_QUERY,
/** */
TOPIC_TX,
/** */
- TOPIC_IO_TEST;
+ TOPIC_IO_TEST,
+
+ /** */
+ TOPIC_HADOOP_MSG;
/** Enum values. */
private static final GridTopic[] VALS = values();
[14/50] [abbrv] ignite git commit: IGNITE-4413 .NET: Fix DateTime
argument handling in SqlQuery
Posted by sb...@apache.org.
IGNITE-4413 .NET: Fix DateTime argument handling in SqlQuery
This closes #1341
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/83710a9d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/83710a9d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/83710a9d
Branch: refs/heads/master
Commit: 83710a9d1bb7379e5f3d891ed95c86096263740b
Parents: 56efb10
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Dec 12 17:52:22 2016 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Dec 12 18:24:23 2016 +0300
----------------------------------------------------------------------
.../Query/CacheQueriesCodeConfigurationTest.cs | 17 ++++++++++--
.../Cache/Query/CacheQueriesTest.cs | 8 ++++++
.../Apache.Ignite.Core/Cache/Query/QueryBase.cs | 15 ++++++++--
.../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 29 +-------------------
4 files changed, 36 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/83710a9d/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
index 7cb999f..863e14f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
@@ -52,7 +52,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Fields = new[]
{
new QueryField("Name", typeof (string)),
- new QueryField("Age", typeof (int))
+ new QueryField("Age", typeof (int)),
+ new QueryField("Birthday", typeof(DateTime)),
},
Indexes = new[]
{
@@ -71,7 +72,8 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
cache[1] = new QueryPerson("Arnold", 10);
cache[2] = new QueryPerson("John", 20);
- using (var cursor = cache.Query(new SqlQuery(typeof (QueryPerson), "age > 10")))
+ using (var cursor = cache.Query(new SqlQuery(typeof (QueryPerson), "age > ? and birthday < ?",
+ 10, DateTime.UtcNow)))
{
Assert.AreEqual(2, cursor.GetAll().Single().Key);
}
@@ -145,7 +147,9 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
cache[2] = new AttributeQueryPerson("John", 20);
- using (var cursor = cache.Query(new SqlQuery(typeof(AttributeQueryPerson), "age > ?", 10)))
+ using (var cursor = cache.Query(new SqlQuery(typeof(AttributeQueryPerson),
+ "age > ? and age < ? and birthday > ? and birthday < ?", 10, 30,
+ DateTime.UtcNow.AddYears(-21), DateTime.UtcNow.AddYears(-19))))
{
Assert.AreEqual(2, cursor.GetAll().Single().Key);
}
@@ -192,6 +196,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Name = name;
Age = age;
Salary = age;
+ Birthday = DateTime.UtcNow.AddYears(-age);
}
/// <summary>
@@ -226,6 +231,12 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
/// </summary>
[QuerySqlField]
public decimal? Salary { get; set; }
+
+ /// <summary>
+ /// Gets or sets the birthday.
+ /// </summary>
+ [QuerySqlField]
+ public DateTime Birthday { get; set; }
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/83710a9d/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
index 49f87c6..20fd93a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+// ReSharper disable UnusedAutoPropertyAccessor.Global
namespace Apache.Ignite.Core.Tests.Cache.Query
{
using System;
@@ -797,6 +798,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
{
Name = name;
Age = age;
+ Birthday = DateTime.UtcNow.AddYears(-age);
}
/// <summary>
@@ -808,6 +810,12 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
/// Age.
/// </summary>
public int Age { get; set; }
+
+ /// <summary>
+ /// Gets or sets the birthday.
+ /// </summary>
+ [QuerySqlField] // Enforce Timestamp serialization
+ public DateTime Birthday { get; set; }
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/83710a9d/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
index cf1f637..d992845 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Query/QueryBase.cs
@@ -17,6 +17,8 @@
namespace Apache.Ignite.Core.Cache.Query
{
+ using System;
+ using Apache.Ignite.Core.Binary;
using Apache.Ignite.Core.Impl.Binary;
using Apache.Ignite.Core.Impl.Cache;
@@ -66,7 +68,7 @@ namespace Apache.Ignite.Core.Cache.Query
/// </summary>
/// <param name="writer">Writer.</param>
/// <param name="args">Arguments.</param>
- internal static void WriteQueryArgs(BinaryWriter writer, object[] args)
+ internal static void WriteQueryArgs(IBinaryRawWriter writer, object[] args)
{
if (args == null)
writer.WriteInt(0);
@@ -75,7 +77,16 @@ namespace Apache.Ignite.Core.Cache.Query
writer.WriteInt(args.Length);
foreach (var arg in args)
- writer.WriteObject(arg);
+ {
+ // Write DateTime as TimeStamp always, otherwise it does not make sense
+ // Wrapped DateTime comparison does not work in SQL
+ var dt = arg as DateTime?; // Works with DateTime also
+
+ if (dt != null)
+ writer.WriteTimestamp(dt);
+ else
+ writer.WriteObject(arg);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/83710a9d/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
index e0d1a3c..186737c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
@@ -980,7 +980,7 @@ namespace Apache.Ignite.Core.Impl.Cache
writer.WriteString(qry.Sql);
writer.WriteInt(qry.PageSize);
- WriteQueryArgs(writer, qry.Arguments);
+ QueryBase.WriteQueryArgs(writer, qry.Arguments);
writer.WriteBoolean(qry.EnableDistributedJoins);
writer.WriteBoolean(qry.EnforceJoinOrder);
@@ -999,33 +999,6 @@ namespace Apache.Ignite.Core.Impl.Cache
return new QueryCursor<TK, TV>(cursor, Marshaller, _flagKeepBinary);
}
- /// <summary>
- /// Write query arguments.
- /// </summary>
- /// <param name="writer">Writer.</param>
- /// <param name="args">Arguments.</param>
- private static void WriteQueryArgs(BinaryWriter writer, object[] args)
- {
- if (args == null)
- writer.WriteInt(0);
- else
- {
- writer.WriteInt(args.Length);
-
- foreach (var arg in args)
- {
- // Write DateTime as TimeStamp always, otherwise it does not make sense
- // Wrapped DateTime comparison does not work in SQL
- var dt = arg as DateTime?; // Works with DateTime also
-
- if (dt != null)
- writer.WriteTimestamp(dt);
- else
- writer.WriteObject(arg);
- }
- }
- }
-
/** <inheritdoc /> */
public IContinuousQueryHandle QueryContinuous(ContinuousQuery<TK, TV> qry)
{
[13/50] [abbrv] ignite git commit: GridClosureProcessor: fixed
javadoc.
Posted by sb...@apache.org.
GridClosureProcessor: fixed javadoc.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8dd4ada6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8dd4ada6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8dd4ada6
Branch: refs/heads/master
Commit: 8dd4ada6287af49a243aba1e58daffa7e44f39a9
Parents: eeb2f2a
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 12 14:56:44 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 12 14:56:44 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/closure/GridClosureProcessor.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8dd4ada6/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 3ed985e..a07dbf8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -599,7 +599,6 @@ public class GridClosureProcessor extends GridProcessorAdapter {
* @param job Closure to execute.
* @param nodes Grid nodes.
* @param sys If {@code true}, then system pool will be used.
- * @param timeout If greater than 0 limits task execution. Cannot be negative.
* @param <R> Type.
* @return Grid future for collection of closure results.
*/
[31/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
index b0353a6..5345a9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiConfigSelfTest.java
@@ -18,8 +18,6 @@
package org.apache.ignite.spi.communication.tcp;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.spi.GridSpiAbstractConfigTest;
import org.apache.ignite.testframework.junits.spi.GridSpiTest;
@@ -47,6 +45,9 @@ public class GridTcpCommunicationSpiConfigSelfTest extends GridSpiAbstractConfig
checkNegativeSpiProperty(new TcpCommunicationSpi(), "ackSendThreshold", 0);
checkNegativeSpiProperty(new TcpCommunicationSpi(), "ackSendThreshold", -1);
checkNegativeSpiProperty(new TcpCommunicationSpi(), "unacknowledgedMessagesBufferSize", -1);
+ checkNegativeSpiProperty(new TcpCommunicationSpi(), "connectionsPerNode", 0);
+ checkNegativeSpiProperty(new TcpCommunicationSpi(), "connectionsPerNode", -1);
+ checkNegativeSpiProperty(new TcpCommunicationSpi(), "connectionsPerNode", Integer.MAX_VALUE);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
index a74a07a..0c57105 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiMultithreadedSelfTest.java
@@ -109,7 +109,7 @@ public class GridTcpCommunicationSpiMultithreadedSelfTest extends GridSpiAbstrac
/**
* @param useShmem Use shared mem.
*/
- protected GridTcpCommunicationSpiMultithreadedSelfTest(boolean useShmem) {
+ GridTcpCommunicationSpiMultithreadedSelfTest(boolean useShmem) {
super(false);
this.useShmem = useShmem;
@@ -370,17 +370,17 @@ public class GridTcpCommunicationSpiMultithreadedSelfTest extends GridSpiAbstrac
Collection<? extends GridNioSession> sessions = GridTestUtils.getFieldValue(srv, "sessions");
for (GridNioSession ses : sessions) {
- final GridNioRecoveryDescriptor snd = ses.recoveryDescriptor();
+ final GridNioRecoveryDescriptor snd = ses.outRecoveryDescriptor();
if (snd != null) {
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
- return snd.messagesFutures().isEmpty();
+ return snd.messagesRequests().isEmpty();
}
}, 10_000);
- assertEquals("Unexpected messages: " + snd.messagesFutures(), 0,
- snd.messagesFutures().size());
+ assertEquals("Unexpected messages: " + snd.messagesRequests(), 0,
+ snd.messagesRequests().size());
}
}
}
@@ -547,11 +547,18 @@ public class GridTcpCommunicationSpiMultithreadedSelfTest extends GridSpiAbstrac
}
for (CommunicationSpi spi : spis.values()) {
- final ConcurrentMap<UUID, GridCommunicationClient> clients = U.field(spi, "clients");
+ final ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(spi, "clients");
assert GridTestUtils.waitForCondition(new PA() {
@Override public boolean apply() {
- return clients.isEmpty();
+ for (GridCommunicationClient[] clients0 : clients.values()) {
+ for (GridCommunicationClient client : clients0) {
+ if (client != null)
+ return false;
+ }
+ }
+
+ return true;
}
}, getTestTimeout()) : "Clients: " + clients;
}
@@ -583,4 +590,4 @@ public class GridTcpCommunicationSpiMultithreadedSelfTest extends GridSpiAbstrac
spis.clear();
nodes.clear();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
index 34872c6..12c2edb 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryAckSelfTest.java
@@ -173,7 +173,7 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
boolean found = false;
for (GridNioSession ses : sessions) {
- final GridNioRecoveryDescriptor recoveryDesc = ses.recoveryDescriptor();
+ final GridNioRecoveryDescriptor recoveryDesc = ses.outRecoveryDescriptor();
if (recoveryDesc != null) {
found = true;
@@ -188,12 +188,12 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
- return recoveryDesc.messagesFutures().isEmpty();
+ return recoveryDesc.messagesRequests().isEmpty();
}
}, 10_000);
- assertEquals("Unexpected messages: " + recoveryDesc.messagesFutures(), 0,
- recoveryDesc.messagesFutures().size());
+ assertEquals("Unexpected messages: " + recoveryDesc.messagesRequests(), 0,
+ recoveryDesc.messagesRequests().size());
break;
}
@@ -361,6 +361,7 @@ public class GridTcpCommunicationSpiRecoveryAckSelfTest<T extends CommunicationS
spi.setAckSendThreshold(ackCnt);
spi.setMessageQueueLimit(queueLimit);
spi.setSharedMemoryPort(-1);
+ spi.setConnectionsPerNode(1);
return spi;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
index 95c9e40..b1aa119 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest.java
@@ -33,6 +33,7 @@ public class GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest extends Gri
spi.setAckSendThreshold(5);
spi.setSocketSendBuffer(512);
spi.setSocketReceiveBuffer(512);
+ spi.setConnectionsPerNode(1);
return spi;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java
new file mode 100644
index 0000000..8e43937
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+/**
+ *
+ */
+public class GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest extends GridTcpCommunicationSpiRecoverySelfTest {
+ /** {@inheritDoc} */
+ @Override protected boolean usePairedConnections() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
index 3234d74..065a3d7 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiRecoverySelfTest.java
@@ -56,6 +56,7 @@ import org.eclipse.jetty.util.ConcurrentHashSet;
/**
*
*/
+@SuppressWarnings("unchecked")
@GridSpiTest(spi = TcpCommunicationSpi.class, group = "Communication SPI")
public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi> extends GridSpiAbstractTest<T> {
/** */
@@ -305,14 +306,14 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
log.info("Iteration: " + i);
try {
- final GridNioSession ses0 = communicationSession(spi0);
- final GridNioSession ses1 = communicationSession(spi1);
+ final GridNioSession ses0 = communicationSession(spi0, false);
+ final GridNioSession ses1 = communicationSession(spi1, true);
ses1.pauseReads().get();
IgniteInternalFuture<?> sndFut = GridTestUtils.runAsync(new Callable<Void>() {
@Override public Void call() throws Exception {
- for (int i = 0; i < 5000; i++) {
+ for (int i = 0; i < 6000; i++) {
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
sentCnt.incrementAndGet();
@@ -331,7 +332,12 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
assertTrue("Failed to wait for session close", ses0.closeTime() != 0);
- ses1.resumeReads().get();
+ try {
+ ses1.resumeReads().get();
+ }
+ catch (IgniteCheckedException ignore) {
+ // Can fail is ses1 was closed.
+ }
for (int j = 0; j < 100; j++) {
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
@@ -415,14 +421,14 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
log.info("Iteration: " + i);
try {
- final GridNioSession ses0 = communicationSession(spi0);
- final GridNioSession ses1 = communicationSession(spi1);
+ final GridNioSession ses0 = communicationSession(spi0, false);
+ final GridNioSession ses1 = communicationSession(spi1, true);
ses1.pauseReads().get();
IgniteInternalFuture<?> sndFut = GridTestUtils.runAsync(new Callable<Void>() {
@Override public Void call() throws Exception {
- for (int i = 0; i < 5000; i++) {
+ for (int i = 0; i < 6000; i++) {
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
expCnt1.incrementAndGet();
@@ -441,12 +447,16 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
assertTrue("Failed to wait for session close", ses0.closeTime() != 0);
- ses1.resumeReads().get();
+ try {
+ ses1.resumeReads().get();
+ }
+ catch (IgniteCheckedException ignore) {
+ // Can fail is ses1 was closed.
+ }
// Wait when session is closed, then try to open new connection from node1.
GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override
- public boolean apply() {
+ @Override public boolean apply() {
return ses1.closeTime() != 0;
}
}, awaitForSocketWriteTimeout());
@@ -532,14 +542,14 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
log.info("Iteration: " + i);
try {
- final GridNioSession ses0 = communicationSession(spi0);
- final GridNioSession ses1 = communicationSession(spi1);
+ final GridNioSession ses0 = communicationSession(spi0, false);
+ final GridNioSession ses1 = communicationSession(spi1, true);
ses1.pauseReads().get();
IgniteInternalFuture<?> sndFut = GridTestUtils.runAsync(new Callable<Void>() {
@Override public Void call() throws Exception {
- for (int i = 0; i < 5000; i++) {
+ for (int i = 0; i < 6000; i++) {
spi0.sendMessage(node1, new GridTestMessage(node0.id(), msgId.incrementAndGet(), 0));
sentCnt.incrementAndGet();
@@ -558,7 +568,12 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
assertTrue("Failed to wait for session close", ses0.closeTime() != 0);
- ses1.resumeReads().get();
+ try {
+ ses1.resumeReads().get();
+ }
+ catch (IgniteCheckedException ignore) {
+ // Can fail is ses1 was closed.
+ }
sndFut.get();
@@ -605,11 +620,12 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
/**
* @param spi SPI.
+ * @param in {@code True} if need find inbound session.
* @return Session.
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
- private GridNioSession communicationSession(TcpCommunicationSpi spi) throws Exception {
+ private GridNioSession communicationSession(TcpCommunicationSpi spi, boolean in) throws Exception {
final GridNioServer srv = U.field(spi, "nioSrvr");
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@@ -622,9 +638,21 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
Collection<? extends GridNioSession> sessions = GridTestUtils.getFieldValue(srv, "sessions");
- assertEquals(1, sessions.size());
+ for (GridNioSession ses : sessions) {
+ if (in == ses.accepted())
+ return ses;
+ }
+
+ fail("Failed to find session");
- return sessions.iterator().next();
+ return null;
+ }
+
+ /**
+ * @return {@code True}.
+ */
+ protected boolean usePairedConnections() {
+ return true;
}
/**
@@ -642,6 +670,8 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
spi.setSocketWriteTimeout(1000);
spi.setSocketSendBuffer(512);
spi.setSocketReceiveBuffer(512);
+ spi.setConnectionsPerNode(1);
+ spi.setUsePairedConnections(usePairedConnections());
return spi;
}
@@ -748,9 +778,8 @@ public class GridTcpCommunicationSpiRecoverySelfTest<T extends CommunicationSpi>
spi.spiStop();
}
- for (IgniteTestResources rsrcs : spiRsrcs) {
+ for (IgniteTestResources rsrcs : spiRsrcs)
rsrcs.stopThreads();
- }
spis.clear();
nodes.clear();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
index 25e3611..c4930a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/IgniteTcpCommunicationRecoveryAckClosureSelfTest.java
@@ -187,7 +187,7 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
boolean found = false;
for (GridNioSession ses : sessions) {
- final GridNioRecoveryDescriptor recoveryDesc = ses.recoveryDescriptor();
+ final GridNioRecoveryDescriptor recoveryDesc = ses.outRecoveryDescriptor();
if (recoveryDesc != null) {
found = true;
@@ -202,12 +202,12 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
- return recoveryDesc.messagesFutures().isEmpty();
+ return recoveryDesc.messagesRequests().isEmpty();
}
}, 10_000);
- assertEquals("Unexpected messages: " + recoveryDesc.messagesFutures(), 0,
- recoveryDesc.messagesFutures().size());
+ assertEquals("Unexpected messages: " + recoveryDesc.messagesRequests(), 0,
+ recoveryDesc.messagesRequests().size());
break;
}
@@ -397,6 +397,7 @@ public class IgniteTcpCommunicationRecoveryAckClosureSelfTest<T extends Communic
spi.setAckSendThreshold(ackCnt);
spi.setMessageQueueLimit(queueLimit);
spi.setSharedMemoryPort(-1);
+ spi.setConnectionsPerNode(1);
return spi;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index f9e2ff4..143159d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -63,7 +63,9 @@ public class GridTestKernalContext extends GridKernalContextImpl {
null,
null,
null,
- U.allPluginProviders());
+ null,
+ U.allPluginProviders()
+ );
GridTestUtils.setFieldValue(grid(), "cfg", config());
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
index 14e5833..17757ab 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
@@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAto
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderFairAffinityMultiNodeFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderMultiNodeFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderOffHeapFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWriteOrderOffHeapTieredFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPrimaryWrityOrderOffHeapMultiNodeFullApiSelfTest;
@@ -77,6 +78,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePar
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOffHeapFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOffHeapMultiNodeFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedOffHeapTieredFullApiSelfTest;
@@ -228,6 +230,10 @@ public class IgniteCacheFullApiSelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheReplicatedFullApiMultithreadedSelfTest.class);
suite.addTestSuite(GridCachePartitionedFullApiMultithreadedSelfTest.class);
+ // Disabled striped pool.
+ suite.addTestSuite(GridCacheAtomicPrimaryWriteOrderNoStripedPoolMultiNodeFullApiSelfTest.class);
+ suite.addTestSuite(GridCachePartitionedNoStripedPoolMultiNodeFullApiSelfTest.class);
+
// Other.
suite.addTestSuite(GridCacheClearSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 435fcfb..554bb3d 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -39,6 +39,10 @@ import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreOptimizedMarshallerW
import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceMultipleConnectionsTest;
+import org.apache.ignite.internal.managers.communication.IgniteCommunicationBalanceTest;
+import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
+import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
@@ -128,7 +132,10 @@ import org.apache.ignite.internal.processors.cache.context.IgniteCacheTxExecutio
import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicNearUpdateTopologyChangeTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheTxNearUpdateTopologyChangeTest;
import org.apache.ignite.internal.processors.cache.distributed.GridCacheEntrySetIterationPreloadingSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecovery10ConnectionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicMessageRecoveryTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecovery10ConnectionsTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheConnectionRecoveryTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheMessageRecoveryIdleConnectionTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheMessageWriteTimeoutTest;
@@ -259,7 +266,7 @@ public class IgniteCacheTestSuite extends TestSuite {
suite.addTestSuite(GridCacheStoreValueBytesSelfTest.class);
GridTestUtils.addTestIfNeeded(suite, DataStreamProcessorSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, DataStreamerUpdateAfterLoadTest.class, ignoredTests);
- suite.addTestSuite(DataStreamerMultiThreadedSelfTest.class);
+ suite.addTestSuite(DataStreamerMultiThreadedSelfTest.class);
suite.addTestSuite(DataStreamerMultinodeCreateCacheTest.class);
suite.addTestSuite(DataStreamerImplSelfTest.class);
suite.addTestSuite(DataStreamerTimeoutTest.class);
@@ -292,10 +299,13 @@ public class IgniteCacheTestSuite extends TestSuite {
suite.addTestSuite(GridCacheEntrySetIterationPreloadingSelfTest.class);
suite.addTestSuite(GridCacheMixedPartitionExchangeSelfTest.class);
suite.addTestSuite(IgniteCacheAtomicMessageRecoveryTest.class);
+ suite.addTestSuite(IgniteCacheAtomicMessageRecoveryNoPairedConnectionsTest.class);
+ suite.addTestSuite(IgniteCacheAtomicMessageRecovery10ConnectionsTest.class);
suite.addTestSuite(IgniteCacheTxMessageRecoveryTest.class);
suite.addTestSuite(IgniteCacheMessageWriteTimeoutTest.class);
suite.addTestSuite(IgniteCacheMessageRecoveryIdleConnectionTest.class);
suite.addTestSuite(IgniteCacheConnectionRecoveryTest.class);
+ suite.addTestSuite(IgniteCacheConnectionRecovery10ConnectionsTest.class);
GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapTieredEvictionAtomicSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapTieredEvictionSelfTest.class, ignoredTests);
GridTestUtils.addTestIfNeeded(suite, GridCacheOffHeapTieredAtomicSelfTest.class, ignoredTests);
@@ -325,6 +335,11 @@ public class IgniteCacheTestSuite extends TestSuite {
suite.addTestSuite(CacheTxFastFinishTest.class);
+ suite.addTestSuite(IgniteVariousConnectionNumberTest.class);
+ suite.addTestSuite(IgniteCommunicationBalanceTest.class);
+ suite.addTestSuite(IgniteCommunicationBalanceMultipleConnectionsTest.class);
+ suite.addTestSuite(IgniteIoTestMessagesTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index c557fbb..11fcfda 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -25,6 +25,7 @@ import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiMultithrea
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiMultithreadedShmemTest;
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoveryAckSelfTest;
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoveryFailureDetectionSelfTest;
+import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest;
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoverySelfTest;
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiRecoverySslSelfTest;
import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiShmemSelfTest;
@@ -50,6 +51,7 @@ public class IgniteSpiCommunicationSelfTestSuite extends TestSuite {
suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoveryAckSelfTest.class));
suite.addTest(new TestSuite(IgniteTcpCommunicationRecoveryAckClosureSelfTest.class));
suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoverySelfTest.class));
+ suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoveryNoPairedConnectionsTest.class));
suite.addTest(new TestSuite(GridTcpCommunicationSpiRecoverySslSelfTest.class));
suite.addTest(new TestSuite(GridTcpCommunicationSpiConcurrentConnectSelfTest.class));
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
index a725534..9542372 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/jobtracker/HadoopJobTracker.java
@@ -240,7 +240,7 @@ public class HadoopJobTracker extends HadoopComponent {
try {
// Must process query callback in a separate thread to avoid deadlocks.
- evtProcSvc.submit(new EventHandler() {
+ evtProcSvc.execute(new EventHandler() {
@Override protected void body() throws IgniteCheckedException {
processJobMetadataUpdates(evts);
}
@@ -264,7 +264,7 @@ public class HadoopJobTracker extends HadoopComponent {
try {
// Must process discovery callback in a separate thread to avoid deadlock.
- evtProcSvc.submit(new EventHandler() {
+ evtProcSvc.execute(new EventHandler() {
@Override protected void body() {
processNodeLeft((DiscoveryEvent)evt);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
index bc047e7..ff58509 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopExternalCommunication.java
@@ -639,6 +639,7 @@ public class HadoopExternalCommunication {
.logger(log.getLogger(GridNioServer.class))
.selectorCount(selectorsCnt)
.gridName(gridName)
+ .serverName("hadoop")
.tcpNoDelay(tcpNoDelay)
.directBuffer(directBuf)
.byteOrder(ByteOrder.nativeOrder())
@@ -1305,11 +1306,11 @@ public class HadoopExternalCommunication {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
if (ses.meta(PROCESS_META) == null && !(msg instanceof ProcessHandshakeMessage))
log.warning("Writing message before handshake has finished [ses=" + ses + ", msg=" + msg + ']');
- return proceedSessionWrite(ses, msg);
+ return proceedSessionWrite(ses, msg, fut);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
index a8de999..3f33fb7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopIpcToNioAdapter.java
@@ -190,7 +190,7 @@ public class HadoopIpcToNioAdapter<T> {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) {
assert ses == HadoopIpcToNioAdapter.this.ses : "ses=" + ses +
", this.ses=" + HadoopIpcToNioAdapter.this.ses;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
index eeca564..24bba88 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/communication/HadoopMarshallerFilter.java
@@ -57,10 +57,10 @@ public class HadoopMarshallerFilter extends GridNioFilterAdapter {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
assert msg instanceof HadoopMessage : "Invalid message type: " + msg;
- return proceedSessionWrite(ses, U.marshal(marsh, msg));
+ return proceedSessionWrite(ses, U.marshal(marsh, msg), fut);
}
@Override public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
@@ -84,4 +84,4 @@ public class HadoopMarshallerFilter extends GridNioFilterAdapter {
@Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
proceedSessionWriteTimeout(ses);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
----------------------------------------------------------------------
diff --git a/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java b/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
index f46ee93..221538c 100644
--- a/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
+++ b/modules/kafka/src/main/java/org/apache/ignite/stream/kafka/KafkaStreamer.java
@@ -169,7 +169,7 @@ public class KafkaStreamer<T, K, V> extends StreamAdapter<T, K, V> {
// Now create an object to consume the messages.
for (final KafkaStream<K, V> stream : streams) {
- executor.submit(new Runnable() {
+ executor.execute(new Runnable() {
@Override public void run() {
while (!stopped) {
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
----------------------------------------------------------------------
diff --git a/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java b/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
index 0b10a75..369e318 100644
--- a/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
+++ b/modules/tools/src/main/java/org/apache/ignite/tools/classgen/ClassesGenerator.java
@@ -122,7 +122,11 @@ public class ClassesGenerator {
for (String err : errs)
sb.append(" ").append(err).append('\n');
- throw new Exception(sb.toString().trim());
+ String msg = sb.toString().trim();
+
+ System.out.println(msg);
+
+ throw new Exception(msg);
}
PrintStream out = new PrintStream(new File(basePath,
@@ -246,4 +250,4 @@ public class ClassesGenerator {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
index 54ef00d..07549d5 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
@@ -30,7 +30,7 @@ import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
import org.apache.ignite.transactions.TransactionOptimisticException;
import org.apache.ignite.transactions.TransactionRollbackException;
-import org.apache.ignite.yardstick.cache.IgniteSqlQueryBenchmark;
+import org.apache.ignite.yardstick.cache.IgnitePutBenchmark;
import org.yardstickframework.BenchmarkDriver;
import org.yardstickframework.BenchmarkDriverStartUp;
@@ -89,13 +89,13 @@ public class IgniteBenchmarkUtils {
public static void main(String[] args) throws Exception {
final String cfg = "modules/yardstick/config/ignite-localhost-config.xml";
- final Class<? extends BenchmarkDriver> benchmark = IgniteSqlQueryBenchmark.class;
+ final Class<? extends BenchmarkDriver> benchmark = IgnitePutBenchmark.class;
final int threads = 1;
final boolean clientDriverNode = true;
- final int extraNodes = 2;
+ final int extraNodes = 4;
final int warmUp = 5;
final int duration = 5;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
index a25f975..c8022e2 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
@@ -72,7 +72,7 @@ public class CacheEntryEventProbe implements BenchmarkProbe {
buildingService = Executors.newSingleThreadExecutor();
- buildingService.submit(new Runnable() {
+ buildingService.execute(new Runnable() {
@Override public void run() {
try {
while (!Thread.currentThread().isInterrupted()) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
new file mode 100644
index 0000000..bee45e0
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteIoTestBenchmark.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public class IgniteIoTestBenchmark extends IgniteAbstractBenchmark {
+ /** */
+ private List<ClusterNode> targetNodes;
+
+ /** */
+ private IgniteKernal ignite;
+
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ ignite = (IgniteKernal)ignite();
+
+ targetNodes = new ArrayList<>();
+
+ ClusterNode loc = ignite().cluster().localNode();
+
+ Collection<ClusterNode> nodes = ignite().cluster().forServers().nodes();
+
+ for (ClusterNode node : nodes) {
+ if (!loc.equals(node))
+ targetNodes.add(node);
+ }
+
+ if (targetNodes.isEmpty())
+ throw new IgniteException("Failed to find remote server nodes [nodes=" + nodes + ']');
+
+ BenchmarkUtils.println(cfg, "Initialized target nodes: " + targetNodes + ']');
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ ClusterNode node = targetNodes.get(nextRandom(targetNodes.size()));
+
+ ignite.sendIoTest(node, null, false).get();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java
new file mode 100644
index 0000000..8791c83
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestAbstractBenchmark.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.io;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ *
+ */
+public abstract class IgniteIoTestAbstractBenchmark extends IgniteAbstractBenchmark {
+ /** */
+ protected final List<ClusterNode> targetNodes = new ArrayList<>();
+
+ /** */
+ protected IgniteKernal ignite;
+
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ ignite = (IgniteKernal)ignite();
+
+ ClusterNode loc = ignite().cluster().localNode();
+
+ Collection<ClusterNode> nodes = ignite().cluster().forServers().nodes();
+
+ for (ClusterNode node : nodes) {
+ if (!loc.equals(node))
+ targetNodes.add(node);
+ }
+
+ if (targetNodes.isEmpty())
+ throw new IgniteException("Failed to find remote server nodes [nodes=" + nodes + ']');
+
+ BenchmarkUtils.println(cfg, "Initialized target nodes: " + F.nodeIds(targetNodes) + ']');
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java
new file mode 100644
index 0000000..9011910
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendAllBenchmark.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.io;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public class IgniteIoTestSendAllBenchmark extends IgniteIoTestAbstractBenchmark {
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ ignite.sendIoTest(targetNodes, null, false).get();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java
new file mode 100644
index 0000000..88368e0
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/io/IgniteIoTestSendRandomBenchmark.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.io;
+
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ *
+ */
+public class IgniteIoTestSendRandomBenchmark extends IgniteIoTestAbstractBenchmark {
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ ClusterNode node = targetNodes.get(nextRandom(targetNodes.size()));
+
+ ignite.sendIoTest(node, null, false).get();
+
+ return true;
+ }
+}
[07/50] [abbrv] ignite git commit: IGNITE-4063: Preserved order of
fields in the metadata according with schema. This closes #1270.
Posted by sb...@apache.org.
IGNITE-4063: Preserved order of fields in the metadata according with schema. This closes #1270.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c143bc1a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c143bc1a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c143bc1a
Branch: refs/heads/master
Commit: c143bc1a77baa13f61d6ba00509fa1fcb33757b1
Parents: 6e48563
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Fri Dec 9 16:05:03 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Dec 9 16:05:03 2016 +0300
----------------------------------------------------------------------
.../internal/binary/BinaryClassDescriptor.java | 12 ++--
.../ignite/internal/binary/BinaryUtils.java | 10 ++-
.../binary/builder/BinaryObjectBuilderImpl.java | 11 +++-
.../platform/PlatformContextImpl.java | 2 +-
.../platform/utils/PlatformUtils.java | 28 +++++++++
.../binary/BinaryMarshallerSelfTest.java | 66 ++++++++++++++++++++
6 files changed, 119 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c143bc1a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
index b121337..5ec519a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
@@ -28,7 +28,6 @@ import java.sql.Timestamp;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -275,15 +274,20 @@ public class BinaryClassDescriptor {
case OBJECT:
// Must not use constructor to honor transient fields semantics.
ctor = null;
- stableFieldsMeta = metaDataEnabled ? new HashMap<String, Integer>() : null;
Map<Object, BinaryFieldAccessor> fields0;
- if (BinaryUtils.FIELDS_SORTED_ORDER)
+ if (BinaryUtils.FIELDS_SORTED_ORDER) {
fields0 = new TreeMap<>();
- else
+
+ stableFieldsMeta = metaDataEnabled ? new TreeMap<String, Integer>() : null;
+ }
+ else {
fields0 = new LinkedHashMap<>();
+ stableFieldsMeta = metaDataEnabled ? new LinkedHashMap<String, Integer>() : null;
+ }
+
Set<String> duplicates = duplicateFields(cls);
Collection<String> names = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c143bc1a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index b304082..bbf5021 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -947,11 +947,17 @@ public class BinaryUtils {
}
// Check and merge fields.
- boolean changed = false;
+ Map<String, Integer> mergedFields;
+
+ if (FIELDS_SORTED_ORDER)
+ mergedFields = new TreeMap<>(oldMeta.fieldsMap());
+ else
+ mergedFields = new LinkedHashMap<>(oldMeta.fieldsMap());
- Map<String, Integer> mergedFields = new HashMap<>(oldMeta.fieldsMap());
Map<String, Integer> newFields = newMeta.fieldsMap();
+ boolean changed = false;
+
for (Map.Entry<String, Integer> newField : newFields.entrySet()) {
Integer oldFieldType = mergedFields.put(newField.getKey(), newField.getValue());
http://git-wip-us.apache.org/repos/asf/ignite/blob/c143bc1a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index 6ea9e7e..68a0ff3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -400,8 +400,12 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder {
if (oldFldTypeName == null) {
// It's a new field, we have to add it to metadata.
- if (fieldsMeta == null)
- fieldsMeta = new HashMap<>();
+ if (fieldsMeta == null) {
+ if (BinaryUtils.FIELDS_SORTED_ORDER)
+ fieldsMeta = new TreeMap<>();
+ else
+ fieldsMeta = new LinkedHashMap<>();
+ }
fieldsMeta.put(name, newFldTypeId);
}
@@ -532,11 +536,12 @@ public class BinaryObjectBuilderImpl implements BinaryObjectBuilder {
@Override public BinaryObjectBuilder setField(String name, Object val0) {
Object val = val0 == null ? new BinaryValueWithType(BinaryUtils.typeByClass(Object.class), null) : val0;
- if (assignedVals == null)
+ if (assignedVals == null) {
if (BinaryUtils.FIELDS_SORTED_ORDER)
assignedVals = new TreeMap<>();
else
assignedVals = new LinkedHashMap<>();
+ }
Object oldVal = assignedVals.put(name, val);
http://git-wip-us.apache.org/repos/asf/ignite/blob/c143bc1a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
index e7fdb0a..6cec7a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
@@ -361,7 +361,7 @@ public class PlatformContextImpl implements PlatformContext {
String typeName = reader.readString();
String affKey = reader.readString();
- Map<String, Integer> fields = PlatformUtils.readMap(reader,
+ Map<String, Integer> fields = PlatformUtils.readLinkedMap(reader,
new PlatformReaderBiClosure<String, Integer>() {
@Override public IgniteBiTuple<String, Integer> read(BinaryRawReaderEx reader) {
return F.t(reader.readString(), reader.readInt());
http://git-wip-us.apache.org/repos/asf/ignite/blob/c143bc1a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index 0d30ad9..959ff68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -388,6 +388,34 @@ public class PlatformUtils {
}
/**
+ * Read linked map.
+ *
+ * @param reader Reader.
+ * @param readClo Reader closure.
+ * @return Map.
+ */
+ public static <K, V> Map<K, V> readLinkedMap(BinaryRawReaderEx reader,
+ @Nullable PlatformReaderBiClosure<K, V> readClo) {
+ int cnt = reader.readInt();
+
+ Map<K, V> map = U.newLinkedHashMap(cnt);
+
+ if (readClo == null) {
+ for (int i = 0; i < cnt; i++)
+ map.put((K)reader.readObjectDetached(), (V)reader.readObjectDetached());
+ }
+ else {
+ for (int i = 0; i < cnt; i++) {
+ IgniteBiTuple<K, V> entry = readClo.read(reader);
+
+ map.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return map;
+ }
+
+ /**
* Read nullable map.
*
* @param reader Reader.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c143bc1a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 39a4d32..6d07c9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -53,6 +53,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
import junit.framework.Assert;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.binary.BinaryBasicIdMapper;
import org.apache.ignite.binary.BinaryBasicNameMapper;
import org.apache.ignite.binary.BinaryCollectionFactory;
@@ -3108,6 +3109,71 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest {
assertNotEquals(binObj02, binObj11);
}
+
+ /**
+ * The test must be refactored after {@link IgniteSystemProperties#IGNITE_BINARY_SORT_OBJECT_FIELDS}
+ * is removed.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFieldOrder() throws Exception {
+ if (BinaryUtils.FIELDS_SORTED_ORDER)
+ return;
+
+ BinaryMarshaller m = binaryMarshaller();
+
+ BinaryObjectImpl binObj = marshal(simpleObject(), m);
+
+ Collection<String> fieldsBin = binObj.type().fieldNames();
+
+ Field[] fields = SimpleObject.class.getDeclaredFields();
+
+ assertEquals(fields.length, fieldsBin.size());
+
+ int i = 0;
+
+ for (String fieldName : fieldsBin) {
+ assertEquals(fields[i].getName(), fieldName);
+
+ ++i;
+ }
+ }
+
+ /**
+ * The test must be refactored after {@link IgniteSystemProperties#IGNITE_BINARY_SORT_OBJECT_FIELDS}
+ * is removed.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFieldOrderByBuilder() throws Exception {
+ if (BinaryUtils.FIELDS_SORTED_ORDER)
+ return;
+
+ BinaryMarshaller m = binaryMarshaller();
+
+ BinaryObjectBuilder builder = new BinaryObjectBuilderImpl(binaryContext(m), "MyFakeClass");
+
+ String[] fieldNames = {"field9", "field8", "field0", "field1", "field2"};
+
+ for (String fieldName : fieldNames)
+ builder.setField(fieldName, 0);
+
+ BinaryObject binObj = builder.build();
+
+
+ Collection<String> fieldsBin = binObj.type().fieldNames();
+
+ assertEquals(fieldNames.length, fieldsBin.size());
+
+ int i = 0;
+
+ for (String fieldName : fieldsBin) {
+ assertEquals(fieldNames[i], fieldName);
+
+ ++i;
+ }
+ }
+
/**
* @param obj Instance of the BinaryObjectImpl to offheap marshalling.
* @param marsh Binary marshaller.
[02/50] [abbrv] ignite git commit: ignite-2358 toString() method for
cache store implementations
Posted by sb...@apache.org.
ignite-2358 toString() method for cache store implementations
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/391f4be4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/391f4be4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/391f4be4
Branch: refs/heads/master
Commit: 391f4be4c687a7f325aeec8b727c9c85ca003454
Parents: 671a77a
Author: agura <ag...@apache.org>
Authored: Wed Dec 7 20:11:50 2016 +0300
Committer: agura <ag...@apache.org>
Committed: Thu Dec 8 19:13:11 2016 +0300
----------------------------------------------------------------------
.../cache/store/cassandra/CassandraCacheStore.java | 6 ++++++
.../cache/store/cassandra/datasource/DataSource.java | 9 +++++++++
.../org/apache/ignite/cache/store/CacheStoreAdapter.java | 6 ++++++
.../ignite/cache/store/jdbc/CacheJdbcPojoStore.java | 6 ++++++
.../processors/cache/CacheStoreBalancingWrapper.java | 6 ++++++
.../processors/cache/GridCacheLoaderWriterStore.java | 6 ++++++
.../platform/dotnet/PlatformDotNetCacheStore.java | 11 +++++++++++
7 files changed, 50 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index f7e7917..e8da3a7 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -41,6 +41,7 @@ import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
import org.apache.ignite.cache.store.cassandra.session.ExecutionAssistant;
import org.apache.ignite.cache.store.cassandra.session.GenericBatchExecutionAssistant;
import org.apache.ignite.cache.store.cassandra.session.LoadCacheCustomQueryWorker;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.logger.NullLogger;
@@ -406,4 +407,9 @@ public class CassandraCacheStore<K, V> implements CacheStore<K, V> {
if (ses != null && (storeSes == null || storeSes.transaction() == null))
U.closeQuiet(ses);
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CassandraCacheStore.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
index 1ecb28f..915eebd 100644
--- a/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
+++ b/modules/cassandra/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
@@ -39,6 +39,8 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
import org.apache.ignite.cache.store.cassandra.session.CassandraSessionImpl;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
/**
* Data source abstraction to specify configuration of the Cassandra session to be used.
@@ -54,9 +56,11 @@ public class DataSource {
private ConsistencyLevel writeConsistency;
/** Username to use for authentication. */
+ @GridToStringExclude
private String user;
/** Password to use for authentication. */
+ @GridToStringExclude
private String pwd;
/** Port to use for Cassandra connection. */
@@ -547,4 +551,9 @@ public class DataSource {
private synchronized void invalidate() {
ses = null;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DataSource.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
index eaf5225..f3436f6 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheStoreAdapter.java
@@ -21,6 +21,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import javax.cache.Cache;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lang.IgniteBiPredicate;
@@ -93,4 +94,9 @@ public abstract class CacheStoreAdapter<K, V> implements CacheStore<K, V> {
@Override public void sessionEnd(boolean commit) {
// No-op.
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheStoreAdapter.class, this);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
index b348020..209dd11 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
@@ -35,6 +35,7 @@ import org.apache.ignite.binary.BinaryObjectBuilder;
import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.binary.BinaryObjectEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
@@ -335,6 +336,11 @@ public class CacheJdbcPojoStore<K, V> extends CacheAbstractJdbcStore<K, V> {
}
}
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheJdbcPojoStore.class, this);
+ }
+
/**
* Description of type property.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
index 8992326..ce2330c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStoreBalancingWrapper.java
@@ -31,6 +31,7 @@ import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
@@ -227,6 +228,11 @@ public class CacheStoreBalancingWrapper<K, V> implements CacheStore<K, V> {
delegate.sessionEnd(commit);
}
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheStoreBalancingWrapper.class, this);
+ }
+
/**
*
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
index c497ac0..03beaf0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheLoaderWriterStore.java
@@ -25,6 +25,7 @@ import javax.cache.Cache;
import javax.cache.integration.CacheLoader;
import javax.cache.integration.CacheWriter;
import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
@@ -142,4 +143,9 @@ class GridCacheLoaderWriterStore<K, V> implements CacheStore<K, V>, LifecycleAwa
@Override public void sessionEnd(boolean commit) {
// No-op.
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GridCacheLoaderWriterStore.class, this);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/391f4be4/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
index 3563dd6..7505d48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/dotnet/PlatformDotNetCacheStore.java
@@ -31,7 +31,10 @@ import org.apache.ignite.internal.processors.platform.memory.PlatformOutputStrea
import org.apache.ignite.internal.processors.platform.utils.PlatformUtils;
import org.apache.ignite.internal.util.lang.GridTuple;
import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lifecycle.LifecycleAware;
@@ -98,12 +101,15 @@ public class PlatformDotNetCacheStore<K, V> implements CacheStore<K, V>, Platfor
private Map<String, ?> props;
/** Native factory. */
+ @GridToStringInclude
private final Object nativeFactory;
/** Interop processor. */
+ @GridToStringExclude
protected PlatformContext platformCtx;
/** Pointer to native store. */
+ @GridToStringExclude
protected long ptr;
/**
@@ -441,4 +447,9 @@ public class PlatformDotNetCacheStore<K, V> implements CacheStore<K, V>, Platfor
return res;
}
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PlatformDotNetCacheStore.class, this);
+ }
}
[27/50] [abbrv] ignite git commit: IGNITE-4426: Hadoop: tasks can
share the same classloader. This closes #1344.
Posted by sb...@apache.org.
IGNITE-4426: Hadoop: tasks can share the same classloader. This closes #1344.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/30b869dd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/30b869dd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/30b869dd
Branch: refs/heads/master
Commit: 30b869ddd32db637ee9ea8f13a115dd4bacc52fe
Parents: b44baf1
Author: devozerov <vo...@gridgain.com>
Authored: Wed Dec 14 14:35:29 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:34 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopClassLoader.java | 10 +++++++++
.../processors/hadoop/HadoopJobProperty.java | 8 +++++++
.../processors/hadoop/impl/v2/HadoopV2Job.java | 22 ++++++++++++++++++--
3 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/30b869dd/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index cd94c89..f6c2fa9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -101,6 +101,16 @@ public class HadoopClassLoader extends URLClassLoader implements ClassCache {
}
/**
+ * Classloader name for job.
+ *
+ * @param jobId Job ID.
+ * @return Name.
+ */
+ public static String nameForJob(HadoopJobId jobId) {
+ return "hadoop-job-" + jobId;
+ }
+
+ /**
* Gets name for the task class loader. Task class loader
* @param info The task info.
* @param prefix Get only prefix (without task type and number)
http://git-wip-us.apache.org/repos/asf/ignite/blob/30b869dd/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 1f0ef1b..9e1dede 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -57,6 +57,13 @@ public enum HadoopJobProperty {
JOB_STATUS_POLL_DELAY("ignite.job.status.poll.delay"),
/**
+ * Whether job classloader can be shared between all tasks.
+ * <p>
+ * Defaults to {@code true}.
+ */
+ JOB_SHARED_CLASSLOADER("ignite.job.shared.classloader"),
+
+ /**
* Size in bytes of single memory page which will be allocated for data structures in shuffle.
* <p>
* By default is {@code 32 * 1024}.
@@ -105,6 +112,7 @@ public enum HadoopJobProperty {
*/
SHUFFLE_JOB_THROTTLE("ignite.shuffle.job.throttle");
+
/** Property name. */
private final String propName;
http://git-wip-us.apache.org/repos/asf/ignite/blob/30b869dd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
index 36da410..a24e581 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2Job.java
@@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
import org.apache.ignite.internal.processors.hadoop.HadoopJob;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
import org.apache.ignite.internal.processors.hadoop.HadoopJobInfo;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskType;
@@ -73,6 +74,7 @@ import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.JOB_SHARED_CLASSLOADER;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.jobLocalDir;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.taskLocalDir;
import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.transformException;
@@ -121,6 +123,9 @@ public class HadoopV2Job implements HadoopJob {
/** File system cache map. */
private final HadoopLazyConcurrentMap<FsCacheKey, FileSystem> fsMap = createHadoopLazyConcurrentMap();
+ /** Shared class loader. */
+ private volatile HadoopClassLoader sharedClsLdr;
+
/** Local node ID */
private volatile UUID locNodeId;
@@ -261,8 +266,8 @@ public class HadoopV2Job implements HadoopJob {
// If there is no pooled class, then load new one.
// Note that the classloader identified by the task it was initially created for,
// but later it may be reused for other tasks.
- HadoopClassLoader ldr = new HadoopClassLoader(rsrcMgr.classPath(),
- HadoopClassLoader.nameForTask(info, false), libNames, helper);
+ HadoopClassLoader ldr = sharedClsLdr != null ?
+ sharedClsLdr : createClassLoader(HadoopClassLoader.nameForTask(info, false));
cls = (Class<? extends HadoopTaskContext>)ldr.loadClass(HadoopV2TaskContext.class.getName());
@@ -312,6 +317,9 @@ public class HadoopV2Job implements HadoopJob {
try {
rsrcMgr.prepareJobEnvironment(!external, jobLocalDir(igniteWorkDirectory(), locNodeId, jobId));
+
+ if (HadoopJobProperty.get(jobInfo, JOB_SHARED_CLASSLOADER, true))
+ sharedClsLdr = createClassLoader(HadoopClassLoader.nameForJob(jobId));
}
finally {
HadoopCommonUtils.restoreContextClassLoader(oldLdr);
@@ -454,4 +462,14 @@ public class HadoopV2Job implements HadoopJob {
public FileSystem fileSystem(@Nullable URI uri, Configuration cfg) throws IOException {
return fileSystemForMrUserWithCaching(uri, cfg, fsMap);
}
+
+ /**
+ * Create class loader with the given name.
+ *
+ * @param name Name.
+ * @return Class loader.
+ */
+ private HadoopClassLoader createClassLoader(String name) {
+ return new HadoopClassLoader(rsrcMgr.classPath(), name, libNames, helper);
+ }
}
\ No newline at end of file
[29/50] [abbrv] ignite git commit: IGNITE-4277: Hadoop: better
property naming for "partially raw" comparator.
Posted by sb...@apache.org.
IGNITE-4277: Hadoop: better property naming for "partially raw" comparator.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f8ac0f14
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f8ac0f14
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f8ac0f14
Branch: refs/heads/master
Commit: f8ac0f14986169ee25159734d0a97b08976c5751
Parents: c1ddf21
Author: devozerov <vo...@gridgain.com>
Authored: Thu Dec 15 12:13:10 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:46 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/hadoop/HadoopJobProperty.java | 4 ++--
.../internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java | 2 +-
.../internal/processors/hadoop/impl/HadoopTeraSortTest.java | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8ac0f14/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index 4122eef..9dd430b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -64,9 +64,9 @@ public enum HadoopJobProperty {
JOB_SHARED_CLASSLOADER("ignite.job.shared.classloader"),
/**
- * Fully qualified name of partially-raw comparator which should be used on sorting phase.
+ * Fully qualified name of partially raw comparator which should be used on sorting phase.
*/
- JOB_PARTIAL_RAW_COMPARATOR("ignite.job.partial.raw.comparator"),
+ JOB_PARTIALLY_RAW_COMPARATOR("ignite.job.partially.raw.comparator"),
/**
* Size in bytes of single memory page which will be allocated for data structures in shuffle.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8ac0f14/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
index 42bbec5..e9cae1c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2TaskContext.java
@@ -433,7 +433,7 @@ public class HadoopV2TaskContext extends HadoopTaskContext {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() {
- Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(), null);
+ Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), null);
if (cls == null)
return null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8ac0f14/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
index a016506..b1fa91f 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -165,7 +165,7 @@ public class HadoopTeraSortTest extends HadoopAbstractSelfTest {
jobConf.setBoolean(HadoopJobProperty.SHUFFLE_MAPPER_STRIPED_OUTPUT.propertyName(), true);
- jobConf.set(HadoopJobProperty.JOB_PARTIAL_RAW_COMPARATOR.propertyName(),
+ jobConf.set(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(),
TextPartiallyRawComparator.class.getName());
Job job = setupConfig(jobConf);
[39/50] [abbrv] ignite git commit: ignite-2412 Do not call 'asyncOp'
for synchronous operations
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index a8219b0..4350b3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -450,61 +450,11 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
}
/** {@inheritDoc} */
- @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
- return dht.getAndPutIfAbsent(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
- return dht.getAndPutIfAbsentAsync(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
- return dht.putIfAbsent(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
- return dht.putIfAbsentAsync(key, val);
- }
-
- /** {@inheritDoc} */
@Nullable @Override public V tryGetAndPut(K key, V val) throws IgniteCheckedException {
return dht.tryGetAndPut(key, val);
}
/** {@inheritDoc} */
- @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
- return dht.getAndReplace(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
- return dht.getAndReplaceAsync(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public boolean replace(K key, V val) throws IgniteCheckedException {
- return dht.replace(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
- return dht.replaceAsync(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
- return dht.replace(key, oldVal, newVal);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
- return dht.replaceAsync(key, oldVal, newVal);
- }
-
- /** {@inheritDoc} */
@Override public void putAll(Map<? extends K, ? extends V> m)
throws IgniteCheckedException {
dht.putAll(m);
@@ -569,6 +519,11 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
}
/** {@inheritDoc} */
+ @Override public boolean remove(K key, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
+ return dht.remove(key, filter);
+ }
+
+ /** {@inheritDoc} */
@Override public V getAndRemove(K key) throws IgniteCheckedException {
return dht.getAndRemove(key);
}
@@ -602,16 +557,6 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
}
/** {@inheritDoc} */
- @Override public boolean remove(K key, V val) throws IgniteCheckedException {
- return dht.remove(key, val);
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
- return dht.removeAsync(key, val);
- }
-
- /** {@inheritDoc} */
@Override public void removeAll() throws IgniteCheckedException {
dht.removeAll();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index bc16ff4..a26d2f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -58,7 +58,6 @@ import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.resource.GridResourceIoc;
-import org.apache.ignite.internal.processors.resource.GridResourceProcessor;
import org.apache.ignite.internal.util.F0;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
@@ -108,6 +107,11 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
}
/** {@inheritDoc} */
+ @Override protected void checkJta() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public boolean isLocal() {
return true;
}
@@ -119,9 +123,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public V getAndPut(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
- A.notNull(key, "key", val, "val");
-
+ @Override protected V getAndPut0(K key, V val, @Nullable CacheEntryPredicate filter) throws IgniteCheckedException {
CacheOperationContext opCtx = ctx.operationContextPerCall();
return (V)updateAllInternal(UPDATE,
@@ -138,16 +140,10 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
}
/** {@inheritDoc} */
- @Override public boolean put(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
- A.notNull(key, "key", val, "val");
-
- boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- long start = statsEnabled ? System.nanoTime() : 0L;
-
+ @Override protected boolean put0(K key, V val, CacheEntryPredicate filter) throws IgniteCheckedException {
CacheOperationContext opCtx = ctx.operationContextPerCall();
- boolean res = (Boolean)updateAllInternal(UPDATE,
+ Boolean res = (Boolean)updateAllInternal(UPDATE,
Collections.singleton(key),
Collections.singleton(val),
null,
@@ -159,8 +155,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
ctx.readThrough(),
opCtx != null && opCtx.isKeepBinary());
- if (statsEnabled)
- metrics0().addPutTimeNanos(System.nanoTime() - start);
+ assert res != null;
return res;
}
@@ -168,8 +163,6 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<V> getAndPutAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
- A.notNull(key, "key", val, "val");
-
return updateAllAsync0(F0.asMap(key, val),
null,
null,
@@ -181,8 +174,6 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
- A.notNull(key, "key", val, "val");
-
return updateAllAsync0(F0.asMap(key, val),
null,
null,
@@ -192,65 +183,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
}
/** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public V getAndPutIfAbsent(K key, V val) throws IgniteCheckedException {
- return getAndPut(key, val, ctx.noVal());
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndPutIfAbsentAsync(K key, V val) {
- return getAndPutAsync(key, val, ctx.noVal());
- }
-
- /** {@inheritDoc} */
- @Override public boolean putIfAbsent(K key, V val) throws IgniteCheckedException {
- return put(key, val, ctx.noVal());
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> putIfAbsentAsync(K key, V val) {
- return putAsync(key, val, ctx.noVal());
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public V getAndReplace(K key, V val) throws IgniteCheckedException {
- return getAndPut(key, val, ctx.hasVal());
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<V> getAndReplaceAsync(K key, V val) {
- return getAndPutAsync(key, val, ctx.hasVal());
- }
-
- /** {@inheritDoc} */
- @Override public boolean replace(K key, V val) throws IgniteCheckedException {
- return put(key, val, ctx.hasVal());
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V val) {
- return putAsync(key, val, ctx.hasVal());
- }
-
- /** {@inheritDoc} */
- @Override public boolean replace(K key, V oldVal, V newVal) throws IgniteCheckedException {
- A.notNull(oldVal, "oldVal");
-
- return put(key, newVal, ctx.equalsVal(oldVal));
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) {
- return putAsync(key, newVal, ctx.equalsVal(oldVal));
- }
-
- /** {@inheritDoc} */
- @Override public void putAll(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
- boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- long start = statsEnabled ? System.nanoTime() : 0L;
-
+ @Override protected void putAll0(Map<? extends K, ? extends V> m) throws IgniteCheckedException {
CacheOperationContext opCtx = ctx.operationContextPerCall();
updateAllInternal(UPDATE,
@@ -264,13 +197,10 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
ctx.writeThrough(),
ctx.readThrough(),
opCtx != null && opCtx.isKeepBinary());
-
- if (statsEnabled)
- metrics0().addPutTimeNanos(System.nanoTime() - start);
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> putAllAsync(Map<? extends K, ? extends V> m) {
+ @Override public IgniteInternalFuture<?> putAllAsync0(Map<? extends K, ? extends V> m) {
return updateAllAsync0(m,
null,
null,
@@ -280,8 +210,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
}
/** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public V getAndRemove(K key) throws IgniteCheckedException {
+ @Override protected V getAndRemove0(K key) throws IgniteCheckedException {
CacheOperationContext opCtx = ctx.operationContextPerCall();
return (V)updateAllInternal(DELETE,
@@ -299,13 +228,13 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public IgniteInternalFuture<V> getAndRemoveAsync(K key) {
+ @Override public IgniteInternalFuture<V> getAndRemoveAsync0(K key) {
return removeAllAsync0(Collections.singletonList(key), true, false, null);
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public void removeAll(Collection<? extends K> keys) throws IgniteCheckedException {
+ @Override public void removeAll0(Collection<? extends K> keys) throws IgniteCheckedException {
CacheOperationContext opCtx = ctx.operationContextPerCall();
updateAllInternal(DELETE,
@@ -322,19 +251,13 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
}
/** {@inheritDoc} */
- @Override public IgniteInternalFuture<?> removeAllAsync(Collection<? extends K> keys) {
+ @Override public IgniteInternalFuture<Object> removeAllAsync0(Collection<? extends K> keys) {
return removeAllAsync0(keys, false, false, null).chain(RET2NULL);
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public boolean remove(K key) throws IgniteCheckedException {
- boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- long start = statsEnabled ? System.nanoTime() : 0L;
-
- A.notNull(key, "key");
-
+ @Override public boolean remove0(K key, final CacheEntryPredicate filter) throws IgniteCheckedException {
CacheOperationContext opCtx = ctx.operationContextPerCall();
Boolean rmv = (Boolean)updateAllInternal(DELETE,
@@ -344,50 +267,23 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
expiryPerCall(),
false,
false,
- null,
+ filter,
ctx.writeThrough(),
ctx.readThrough(),
opCtx != null && opCtx.isKeepBinary());
- if (statsEnabled && rmv)
- metrics0().addRemoveTimeNanos(System.nanoTime() - start);
+ assert rmv != null;
return rmv;
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public IgniteInternalFuture<Boolean> removeAsync(K key, @Nullable CacheEntryPredicate filter) {
- A.notNull(key, "key");
-
+ @Override public IgniteInternalFuture<Boolean> removeAsync0(K key, @Nullable CacheEntryPredicate filter) {
return removeAllAsync0(Collections.singletonList(key), false, false, filter);
}
/** {@inheritDoc} */
- @Override public boolean remove(K key, V val) throws IgniteCheckedException {
- A.notNull(key, "key", val, "val");
-
- CacheOperationContext opCtx = ctx.operationContextPerCall();
-
- return (Boolean)updateAllInternal(DELETE,
- Collections.singleton(key),
- null,
- null,
- expiryPerCall(),
- false,
- false,
- ctx.equalsVal(val),
- ctx.writeThrough(),
- ctx.readThrough(),
- opCtx != null && opCtx.isKeepBinary());
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Boolean> removeAsync(K key, V val) {
- return removeAsync(key, ctx.equalsVal(val));
- }
-
- /** {@inheritDoc} */
@Override public IgniteInternalFuture<?> removeAllAsync() {
return ctx.closures().callLocalSafe(new Callable<Void>() {
@Override public Void call() throws Exception {
@@ -399,11 +295,13 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
}
/** {@inheritDoc} */
-
@SuppressWarnings("unchecked")
- @Override @Nullable public V get(K key, boolean deserializeBinary, boolean needVer) throws IgniteCheckedException {
- String taskName = ctx.kernalContext().job().currentTaskName();
-
+ @Override protected V get0(
+ final K key,
+ String taskName,
+ boolean deserializeBinary,
+ boolean needVer) throws IgniteCheckedException
+ {
Map<K, V> m = getAllInternal(Collections.singleton(key),
ctx.isSwapOrOffheapEnabled(),
ctx.readThrough(),
@@ -419,7 +317,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public final Map<K, V> getAll(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
+ @Override public final Map<K, V> getAll0(Collection<? extends K> keys, boolean deserializeBinary, boolean needVer)
throws IgniteCheckedException {
A.notNull(keys, "keys");
@@ -794,7 +692,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
- IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+ return asyncOp(new Callable<Object>() {
@Override public Object call() throws Exception {
return updateAllInternal(op,
keys,
@@ -809,11 +707,6 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
keepBinary);
}
});
-
- if (ctx.config().isStatisticsEnabled())
- fut.listen(new UpdatePutTimeStatClosure(metrics0(), System.nanoTime()));
-
- return fut;
}
/**
@@ -835,17 +728,13 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
final boolean readThrough = ctx.readThrough();
- final boolean statsEnabled = ctx.config().isStatisticsEnabled();
-
- final long start = statsEnabled ? System.nanoTime() : 0L;
-
final ExpiryPolicy expiryPlc = expiryPerCall();
CacheOperationContext opCtx = ctx.operationContextPerCall();
final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
- IgniteInternalFuture fut = asyncOp(new Callable<Object>() {
+ return asyncOp(new Callable<Object>() {
@Override public Object call() throws Exception {
return updateAllInternal(DELETE,
keys,
@@ -860,11 +749,6 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
keepBinary);
}
});
-
- if (statsEnabled)
- fut.listen(new UpdateRemoveTimeStatClosure<>(metrics0(), start));
-
- return fut;
}
/**
@@ -1584,10 +1468,7 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
* @return Future.
*/
@SuppressWarnings("unchecked")
- protected IgniteInternalFuture asyncOp(final Callable<?> op) {
- if (!asyncToggled)
- return ctx.closures().callLocalSafe(op);
-
+ private IgniteInternalFuture asyncOp(final Callable<?> op) {
IgniteInternalFuture fail = asyncOpAcquire();
if (fail != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
index 3e3b84e..648134e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.java
@@ -34,7 +34,7 @@ public class CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest extend
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
- cfg.getTransactionConfiguration().setDefaultTxTimeout(Long.MAX_VALUE);
+ cfg.getTransactionConfiguration().setDefaultTxTimeout(5 * 60_000);
return cfg;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ad785cbd/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
index ec3b808..57c709b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.processors.igfs;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteFileSystem;
@@ -39,14 +41,11 @@ import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.transactions.TransactionConcurrency;
-import org.apache.ignite.transactions.TransactionIsolation;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
/**
* Test to check for system pool starvation due to {@link IgfsBlocksMessage}.
@@ -125,8 +124,7 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst
@Override public Void call() throws Exception {
GridCacheAdapter dataCache = dataCache(attacker);
- try (IgniteInternalTx tx =
- dataCache.txStartEx(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ try (IgniteInternalTx tx = dataCache.txStartEx(PESSIMISTIC, REPEATABLE_READ)) {
dataCache.put(DATA_KEY, 0);
txStartLatch.countDown();
@@ -185,6 +183,7 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst
* Create IGFS file asynchronously.
*
* @param path Path.
+ * @param writeStartLatch Write start latch.
* @return Future.
*/
private IgniteInternalFuture<Void> createFileAsync(final IgfsPath path, final CountDownLatch writeStartLatch) {
@@ -265,6 +264,7 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst
cfg.setLocalHost("127.0.0.1");
cfg.setConnectorConfiguration(null);
+ cfg.setStripedPoolSize(0);
cfg.setSystemThreadPoolSize(2);
cfg.setRebalanceThreadPoolSize(1);
cfg.setPublicThreadPoolSize(1);
[17/50] [abbrv] ignite git commit: IGNITE-4274: Hadoop: added new
property to control shuffle message size.
Posted by sb...@apache.org.
IGNITE-4274: Hadoop: added new property to control shuffle message size.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2488f340
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2488f340
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2488f340
Branch: refs/heads/master
Commit: 2488f340c85301c0ec39cac80a8426f5f4c3caf0
Parents: ae903c5
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 5 15:28:54 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:45:26 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopJobProperty.java | 37 ++++++++++++--------
.../hadoop/shuffle/HadoopShuffleJob.java | 12 +++++--
2 files changed, 32 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2488f340/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
index e7bf565..e713caa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobProperty.java
@@ -28,40 +28,40 @@ public enum HadoopJobProperty {
* <p>
* Setting it right allows to avoid rehashing.
*/
- COMBINER_HASHMAP_SIZE,
+ COMBINER_HASHMAP_SIZE("ignite.combiner.hashmap.size"),
/**
* Initial size for hashmap which stores output of mapper or combiner and will be used as input of reducer.
* <p>
* Setting it right allows to avoid rehashing.
*/
- PARTITION_HASHMAP_SIZE,
+ PARTITION_HASHMAP_SIZE("ignite.partition.hashmap.size"),
/**
* Specifies number of concurrently running mappers for external execution mode.
* <p>
* If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
*/
- EXTERNAL_CONCURRENT_MAPPERS,
+ EXTERNAL_CONCURRENT_MAPPERS("ignite.external.concurrent.mappers"),
/**
* Specifies number of concurrently running reducers for external execution mode.
* <p>
* If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}.
*/
- EXTERNAL_CONCURRENT_REDUCERS,
+ EXTERNAL_CONCURRENT_REDUCERS("ignite.external.concurrent.reducers"),
/**
* Delay in milliseconds after which Ignite server will reply job status.
*/
- JOB_STATUS_POLL_DELAY,
+ JOB_STATUS_POLL_DELAY("ignite.job.status.poll.delay"),
/**
* Size in bytes of single memory page which will be allocated for data structures in shuffle.
* <p>
* By default is {@code 32 * 1024}.
*/
- SHUFFLE_OFFHEAP_PAGE_SIZE,
+ SHUFFLE_OFFHEAP_PAGE_SIZE("ignite.shuffle.offheap.page.size"),
/**
* If set to {@code true} then input for combiner will not be sorted by key.
@@ -71,7 +71,7 @@ public enum HadoopJobProperty {
* <p>
* By default is {@code false}.
*/
- SHUFFLE_COMBINER_NO_SORTING,
+ SHUFFLE_COMBINER_NO_SORTING("ignite.shuffle.combiner.no.sorting"),
/**
* If set to {@code true} then input for reducer will not be sorted by key.
@@ -81,7 +81,14 @@ public enum HadoopJobProperty {
* <p>
* By default is {@code false}.
*/
- SHUFFLE_REDUCER_NO_SORTING,
+ SHUFFLE_REDUCER_NO_SORTING("ignite.shuffle.reducer.no.sorting"),
+
+ /**
+ * Defines approximate size in bytes of shuffle message which will be passed over wire from mapper to reducer.
+ * <p>
+ * Defaults to 128Kb.
+ */
+ SHUFFLE_MSG_SIZE("ignite.shuffle.message.size"),
/**
* Shuffle job throttle in milliseconds. When job is executed with separate shuffle thread, this parameter
@@ -89,23 +96,25 @@ public enum HadoopJobProperty {
* <p>
* Defaults to {@code 0}.
*/
- SHUFFLE_JOB_THROTTLE;
+ SHUFFLE_JOB_THROTTLE("ignite.shuffle.job.throttle");
- /** */
- private final String ptyName;
+ /** Property name. */
+ private final String propName;
/**
+ * Constrcutor.
*
+ * @param propName Property name.
*/
- HadoopJobProperty() {
- ptyName = "ignite." + name().toLowerCase().replace('_', '.');
+ HadoopJobProperty(String propName) {
+ this.propName = propName;
}
/**
* @return Property name.
*/
public String propertyName() {
- return ptyName;
+ return propName;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/2488f340/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 8c731c0..e5af8f1 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -55,6 +55,7 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.thread.IgniteThread;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -64,7 +65,7 @@ import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get
*/
public class HadoopShuffleJob<T> implements AutoCloseable {
/** */
- private static final int MSG_BUF_SIZE = 128 * 1024;
+ private static final int DFLT_SHUFFLE_MSG_SIZE = 128 * 1024;
/** */
private final HadoopJob job;
@@ -109,6 +110,9 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/** */
private final IgniteLogger log;
+ /** Message size. */
+ private final int msgSize;
+
/** */
private final long throttle;
@@ -128,6 +132,8 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
this.mem = mem;
this.log = log.getLogger(HadoopShuffleJob.class);
+ msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
+
if (!F.isEmpty(locReducers)) {
for (int rdc : locReducers) {
HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
@@ -320,7 +326,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
continue; // Skip empty map and local node.
if (msgs[i] == null)
- msgs[i] = new HadoopShuffleMessage(job.id(), i, MSG_BUF_SIZE);
+ msgs[i] = new HadoopShuffleMessage(job.id(), i, msgSize);
final int idx = i;
@@ -425,7 +431,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
});
msgs[idx] = newBufMinSize == 0 ? null : new HadoopShuffleMessage(job.id(), idx,
- Math.max(MSG_BUF_SIZE, newBufMinSize));
+ Math.max(msgSize, newBufMinSize));
}
/** {@inheritDoc} */
[46/50] [abbrv] ignite git commit: Fixed 'singleRmv' flag for
cache.remove(k, v) operation in tx cache.
Posted by sb...@apache.org.
Fixed 'singleRmv' flag for cache.remove(k, v) operation in tx cache.
(cherry picked from commit 126ab60)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/800579c9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/800579c9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/800579c9
Branch: refs/heads/master
Commit: 800579c9ca193f7660e35f241c255904c573eef1
Parents: e82eefe
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 20 15:15:04 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 20 15:18:28 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/processors/cache/GridCacheAdapter.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/800579c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index f235f6a..3b4557d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2948,6 +2948,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/**
* @param key Key.
+ * @param filter Filter.
* @return {@code True} if entry was removed.
* @throws IgniteCheckedException If failed.
*/
@@ -2959,7 +2960,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
Collections.singletonList(key),
/*retval*/false,
filter,
- /*singleRmv*/true).get().success();
+ /*singleRmv*/filter == null).get().success();
}
@Override public String toString() {
[19/50] [abbrv] ignite git commit: IGNITE-4271: Hadoop: shuffle
messages now use "direct-marshallable" path this avoiding unnecessary copying
as opposed to user messages which were used previously. This closes #1266.
This closes #1313.
Posted by sb...@apache.org.
IGNITE-4271: Hadoop: shuffle messages now use "direct-marshallable" path this avoiding unnecessary copying as opposed to user messages which were used previously. This closes #1266. This closes #1313.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b7b97cfd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b7b97cfd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b7b97cfd
Branch: refs/heads/master
Commit: b7b97cfd3419da4421423e189369a4931efd3996
Parents: 3c38421
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 5 15:49:01 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:45:37 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/GridTopic.java | 3 +
.../communication/GridIoMessageFactory.java | 18 +
.../internal/processors/hadoop/HadoopJobId.java | 79 +++-
.../hadoop/message/HadoopMessage.java | 27 ++
.../hadoop/shuffle/HadoopShuffleAck.java | 170 +++++++++
.../hadoop/shuffle/HadoopShuffleMessage.java | 361 +++++++++++++++++++
.../hadoop/message/HadoopMessage.java | 27 --
.../hadoop/shuffle/HadoopShuffle.java | 25 +-
.../hadoop/shuffle/HadoopShuffleAck.java | 92 -----
.../hadoop/shuffle/HadoopShuffleJob.java | 5 +-
.../hadoop/shuffle/HadoopShuffleMessage.java | 242 -------------
.../child/HadoopChildProcessRunner.java | 6 +-
12 files changed, 682 insertions(+), 373 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 248f75b..b5608db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -91,6 +91,9 @@ public enum GridTopic {
TOPIC_HADOOP,
/** */
+ TOPIC_HADOOP_MSG,
+
+ /** */
TOPIC_QUERY,
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index f36191c..dd68984 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -117,6 +117,9 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerResponse;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleAck;
+import org.apache.ignite.internal.processors.hadoop.shuffle.HadoopShuffleMessage;
import org.apache.ignite.internal.processors.igfs.IgfsAckMessage;
import org.apache.ignite.internal.processors.igfs.IgfsBlockKey;
import org.apache.ignite.internal.processors.igfs.IgfsBlocksMessage;
@@ -165,6 +168,21 @@ public class GridIoMessageFactory implements MessageFactory {
Message msg = null;
switch (type) {
+ case -30:
+ msg = new HadoopJobId();
+
+ break;
+
+ case -29:
+ msg = new HadoopShuffleAck();
+
+ break;
+
+ case -28:
+ msg = new HadoopShuffleMessage();
+
+ break;
+
case -27:
msg = new GridDhtTxOnePhaseCommitAckRequest();
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
index 8c61fab..740ab89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobId.java
@@ -21,14 +21,18 @@ import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.ignite.internal.processors.cache.GridCacheInternal;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
/**
* Job ID.
*/
-public class HadoopJobId implements GridCacheInternal, Externalizable {
+public class HadoopJobId implements Message, GridCacheInternal, Externalizable {
/** */
private static final long serialVersionUID = 0L;
@@ -63,6 +67,79 @@ public class HadoopJobId implements GridCacheInternal, Externalizable {
}
/** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeUuid("nodeId", nodeId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeInt("localId", jobId))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ nodeId = reader.readUuid("nodeId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ jobId = reader.readInt("jobId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(HadoopJobId.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -30;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeUuid(out, nodeId);
out.writeInt(jobId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
new file mode 100644
index 0000000..0d7bd3a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.message;
+
+import java.io.Externalizable;
+
+/**
+ * Marker interface for all hadoop messages.
+ */
+public interface HadoopMessage extends Externalizable {
+ // No-op.
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
new file mode 100644
index 0000000..6dd2c2d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+/**
+ * Acknowledgement message.
+ */
+public class HadoopShuffleAck implements HadoopMessage, Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @GridToStringInclude
+ private long msgId;
+
+ /** */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /**
+ *
+ */
+ public HadoopShuffleAck() {
+ // No-op.
+ }
+
+ /**
+ * @param msgId Message ID.
+ */
+ public HadoopShuffleAck(long msgId, HadoopJobId jobId) {
+ assert jobId != null;
+
+ this.msgId = msgId;
+ this.jobId = jobId;
+ }
+
+ /**
+ * @return Message ID.
+ */
+ public long id() {
+ return msgId;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeLong("msgId", msgId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeMessage("jobId", jobId))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ msgId = reader.readLong("msgId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ jobId = reader.readMessage("jobId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(HadoopShuffleAck.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -29;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+ out.writeLong(msgId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+
+ jobId.readExternal(in);
+ msgId = in.readLong();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopShuffleAck.class, this);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
new file mode 100644
index 0000000..3732bc2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.shuffle;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Shuffle message.
+ */
+public class HadoopShuffleMessage implements Message, HadoopMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final AtomicLong ids = new AtomicLong();
+
+ /** */
+ private static final byte MARKER_KEY = (byte)17;
+
+ /** */
+ private static final byte MARKER_VALUE = (byte)31;
+
+ /** */
+ @GridToStringInclude
+ private long msgId;
+
+ /** */
+ @GridToStringInclude
+ private HadoopJobId jobId;
+
+ /** */
+ @GridToStringInclude
+ private int reducer;
+
+ /** */
+ private byte[] buf;
+
+ /** */
+ @GridToStringInclude
+ private int off;
+
+ /**
+ *
+ */
+ public HadoopShuffleMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param size Size.
+ */
+ public HadoopShuffleMessage(HadoopJobId jobId, int reducer, int size) {
+ assert jobId != null;
+
+ buf = new byte[size];
+
+ this.jobId = jobId;
+ this.reducer = reducer;
+
+ msgId = ids.incrementAndGet();
+ }
+
+ /**
+ * @return Message ID.
+ */
+ public long id() {
+ return msgId;
+ }
+
+ /**
+ * @return Job ID.
+ */
+ public HadoopJobId jobId() {
+ return jobId;
+ }
+
+ /**
+ * @return Reducer.
+ */
+ public int reducer() {
+ return reducer;
+ }
+
+ /**
+ * @return Buffer.
+ */
+ public byte[] buffer() {
+ return buf;
+ }
+
+ /**
+ * @return Offset.
+ */
+ public int offset() {
+ return off;
+ }
+
+ /**
+ * @param size Size.
+ * @param valOnly Only value wll be added.
+ * @return {@code true} If this message can fit additional data of this size
+ */
+ public boolean available(int size, boolean valOnly) {
+ size += valOnly ? 5 : 10;
+
+ if (off + size > buf.length) {
+ if (off == 0) { // Resize if requested size is too big.
+ buf = new byte[size];
+
+ return true;
+ }
+
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param keyPtr Key pointer.
+ * @param keySize Key size.
+ */
+ public void addKey(long keyPtr, int keySize) {
+ add(MARKER_KEY, keyPtr, keySize);
+ }
+
+ /**
+ * @param valPtr Value pointer.
+ * @param valSize Value size.
+ */
+ public void addValue(long valPtr, int valSize) {
+ add(MARKER_VALUE, valPtr, valSize);
+ }
+
+ /**
+ * @param marker Marker.
+ * @param ptr Pointer.
+ * @param size Size.
+ */
+ private void add(byte marker, long ptr, int size) {
+ buf[off++] = marker;
+
+ GridUnsafe.putInt(buf, GridUnsafe.BYTE_ARR_OFF + off, size);
+
+ off += 4;
+
+ GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF + off, size);
+
+ off += size;
+ }
+
+ /**
+ * @param v Visitor.
+ */
+ public void visit(Visitor v) throws IgniteCheckedException {
+ for (int i = 0; i < off;) {
+ byte marker = buf[i++];
+
+ int size = GridUnsafe.getInt(buf, GridUnsafe.BYTE_ARR_OFF + i);
+
+ i += 4;
+
+ if (marker == MARKER_VALUE)
+ v.onValue(buf, i, size);
+ else if (marker == MARKER_KEY)
+ v.onKey(buf, i, size);
+ else
+ throw new IllegalStateException();
+
+ i += size;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeLong("msgId", msgId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeMessage("jobId", jobId))
+ return false;
+
+ writer.incrementState();
+
+ case 2:
+ if (!writer.writeInt("reducer", reducer))
+ return false;
+
+ writer.incrementState();
+
+ case 3:
+ if (!writer.writeByteArray("buf", this.buf))
+ return false;
+
+ writer.incrementState();
+
+ case 4:
+ if (!writer.writeInt("off", off))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ msgId = reader.readLong("msgId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ jobId = reader.readMessage("jobId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 2:
+ reducer = reader.readInt("reducer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 3:
+ this.buf = reader.readByteArray("buf");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 4:
+ off = reader.readInt("off");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(HadoopShuffleMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -28;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 5;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ jobId.writeExternal(out);
+ out.writeLong(msgId);
+ out.writeInt(reducer);
+ out.writeInt(off);
+ U.writeByteArray(out, buf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ jobId = new HadoopJobId();
+
+ jobId.readExternal(in);
+ msgId = in.readLong();
+ reducer = in.readInt();
+ off = in.readInt();
+ buf = U.readByteArray(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopShuffleMessage.class, this);
+ }
+
+ /**
+ * Visitor.
+ */
+ public static interface Visitor {
+ /**
+ * @param buf Buffer.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException;
+
+ /**
+ * @param buf Buffer.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public void onValue(byte[] buf, int off, int len) throws IgniteCheckedException;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
deleted file mode 100644
index 0d7bd3a..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/message/HadoopMessage.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.message;
-
-import java.io.Externalizable;
-
-/**
- * Marker interface for all hadoop messages.
- */
-public interface HadoopMessage extends Externalizable {
- // No-op.
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
index 769bdc4..a69e779 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffle.java
@@ -17,13 +17,12 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.GridTopic;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.processors.hadoop.HadoopComponent;
import org.apache.ignite.internal.processors.hadoop.HadoopContext;
import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
@@ -38,6 +37,11 @@ import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
/**
* Shuffle.
@@ -53,6 +57,12 @@ public class HadoopShuffle extends HadoopComponent {
@Override public void start(HadoopContext ctx) throws IgniteCheckedException {
super.start(ctx);
+ ctx.kernalContext().io().addMessageListener(GridTopic.TOPIC_HADOOP_MSG, new GridMessageListener() {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ onMessageReceived(nodeId, (HadoopMessage)msg);
+ }
+ });
+
ctx.kernalContext().io().addUserMessageListener(GridTopic.TOPIC_HADOOP,
new IgniteBiPredicate<UUID, Object>() {
@Override public boolean apply(UUID nodeId, Object msg) {
@@ -117,7 +127,10 @@ public class HadoopShuffle extends HadoopComponent {
private void send0(UUID nodeId, Object msg) throws IgniteCheckedException {
ClusterNode node = ctx.kernalContext().discovery().node(nodeId);
- ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
+ if (msg instanceof Message)
+ ctx.kernalContext().io().send(node, GridTopic.TOPIC_HADOOP_MSG, (Message)msg, GridIoPolicy.PUBLIC_POOL);
+ else
+ ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
}
/**
@@ -151,8 +164,8 @@ public class HadoopShuffle extends HadoopComponent {
*/
private void startSending(HadoopShuffleJob<UUID> shuffleJob) {
shuffleJob.startSending(ctx.kernalContext().gridName(),
- new IgniteInClosure2X<UUID, HadoopShuffleMessage>() {
- @Override public void applyx(UUID dest, HadoopShuffleMessage msg) throws IgniteCheckedException {
+ new IgniteInClosure2X<UUID, HadoopMessage>() {
+ @Override public void applyx(UUID dest, HadoopMessage msg) throws IgniteCheckedException {
send0(dest, msg);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
deleted file mode 100644
index 6013ec6..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleAck.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
-import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-
-/**
- * Acknowledgement message.
- */
-public class HadoopShuffleAck implements HadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- @GridToStringInclude
- private long msgId;
-
- /** */
- @GridToStringInclude
- private HadoopJobId jobId;
-
- /**
- *
- */
- public HadoopShuffleAck() {
- // No-op.
- }
-
- /**
- * @param msgId Message ID.
- */
- public HadoopShuffleAck(long msgId, HadoopJobId jobId) {
- assert jobId != null;
-
- this.msgId = msgId;
- this.jobId = jobId;
- }
-
- /**
- * @return Message ID.
- */
- public long id() {
- return msgId;
- }
-
- /**
- * @return Job ID.
- */
- public HadoopJobId jobId() {
- return jobId;
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
- out.writeLong(msgId);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new HadoopJobId();
-
- jobId.readExternal(in);
- msgId = in.readLong();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HadoopShuffleAck.class, this);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index 025c4da..9392b2c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopTaskInput;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskOutput;
import org.apache.ignite.internal.processors.hadoop.HadoopTaskType;
import org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter;
+import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopConcurrentHashMultimap;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopMultimap;
import org.apache.ignite.internal.processors.hadoop.shuffle.collections.HadoopSkipList;
@@ -92,7 +93,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
private final AtomicReferenceArray<HadoopMultimap> maps;
/** */
- private volatile IgniteInClosure2X<T, HadoopShuffleMessage> io;
+ private volatile IgniteInClosure2X<T, HadoopMessage> io;
/** */
protected ConcurrentMap<Long, IgniteBiTuple<HadoopShuffleMessage, GridFutureAdapter<?>>> sentMsgs =
@@ -176,7 +177,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
* @param io IO Closure for sending messages.
*/
@SuppressWarnings("BusyWait")
- public void startSending(String gridName, IgniteInClosure2X<T, HadoopShuffleMessage> io) {
+ public void startSending(String gridName, IgniteInClosure2X<T, HadoopMessage> io) {
assert snd == null;
assert io != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
deleted file mode 100644
index 71a314b..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.shuffle;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
-import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- * Shuffle message.
- */
-public class HadoopShuffleMessage implements HadoopMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private static final AtomicLong ids = new AtomicLong();
-
- /** */
- private static final byte MARKER_KEY = (byte)17;
-
- /** */
- private static final byte MARKER_VALUE = (byte)31;
-
- /** */
- @GridToStringInclude
- private long msgId;
-
- /** */
- @GridToStringInclude
- private HadoopJobId jobId;
-
- /** */
- @GridToStringInclude
- private int reducer;
-
- /** */
- private byte[] buf;
-
- /** */
- @GridToStringInclude
- private int off;
-
- /**
- *
- */
- public HadoopShuffleMessage() {
- // No-op.
- }
-
- /**
- * @param size Size.
- */
- public HadoopShuffleMessage(HadoopJobId jobId, int reducer, int size) {
- assert jobId != null;
-
- buf = new byte[size];
-
- this.jobId = jobId;
- this.reducer = reducer;
-
- msgId = ids.incrementAndGet();
- }
-
- /**
- * @return Message ID.
- */
- public long id() {
- return msgId;
- }
-
- /**
- * @return Job ID.
- */
- public HadoopJobId jobId() {
- return jobId;
- }
-
- /**
- * @return Reducer.
- */
- public int reducer() {
- return reducer;
- }
-
- /**
- * @return Buffer.
- */
- public byte[] buffer() {
- return buf;
- }
-
- /**
- * @return Offset.
- */
- public int offset() {
- return off;
- }
-
- /**
- * @param size Size.
- * @param valOnly Only value wll be added.
- * @return {@code true} If this message can fit additional data of this size
- */
- public boolean available(int size, boolean valOnly) {
- size += valOnly ? 5 : 10;
-
- if (off + size > buf.length) {
- if (off == 0) { // Resize if requested size is too big.
- buf = new byte[size];
-
- return true;
- }
-
- return false;
- }
-
- return true;
- }
-
- /**
- * @param keyPtr Key pointer.
- * @param keySize Key size.
- */
- public void addKey(long keyPtr, int keySize) {
- add(MARKER_KEY, keyPtr, keySize);
- }
-
- /**
- * @param valPtr Value pointer.
- * @param valSize Value size.
- */
- public void addValue(long valPtr, int valSize) {
- add(MARKER_VALUE, valPtr, valSize);
- }
-
- /**
- * @param marker Marker.
- * @param ptr Pointer.
- * @param size Size.
- */
- private void add(byte marker, long ptr, int size) {
- buf[off++] = marker;
-
- GridUnsafe.putInt(buf, GridUnsafe.BYTE_ARR_OFF + off, size);
-
- off += 4;
-
- GridUnsafe.copyOffheapHeap(ptr, buf, GridUnsafe.BYTE_ARR_OFF + off, size);
-
- off += size;
- }
-
- /**
- * @param v Visitor.
- */
- public void visit(Visitor v) throws IgniteCheckedException {
- for (int i = 0; i < off;) {
- byte marker = buf[i++];
-
- int size = GridUnsafe.getInt(buf, GridUnsafe.BYTE_ARR_OFF + i);
-
- i += 4;
-
- if (marker == MARKER_VALUE)
- v.onValue(buf, i, size);
- else if (marker == MARKER_KEY)
- v.onKey(buf, i, size);
- else
- throw new IllegalStateException();
-
- i += size;
- }
- }
-
- /** {@inheritDoc} */
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- jobId.writeExternal(out);
- out.writeLong(msgId);
- out.writeInt(reducer);
- out.writeInt(off);
- U.writeByteArray(out, buf);
- }
-
- /** {@inheritDoc} */
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- jobId = new HadoopJobId();
-
- jobId.readExternal(in);
- msgId = in.readLong();
- reducer = in.readInt();
- off = in.readInt();
- buf = U.readByteArray(in);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HadoopShuffleMessage.class, this);
- }
-
- /**
- * Visitor.
- */
- public static interface Visitor {
- /**
- * @param buf Buffer.
- * @param off Offset.
- * @param len Length.
- */
- public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException;
-
- /**
- * @param buf Buffer.
- * @param off Offset.
- * @param len Length.
- */
- public void onValue(byte[] buf, int off, int len) throws IgniteCheckedException;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7b97cfd/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
index 45d9a27..7001b8c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/child/HadoopChildProcessRunner.java
@@ -259,9 +259,9 @@ public class HadoopChildProcessRunner {
if (req.reducersAddresses() != null) {
if (shuffleJob.initializeReduceAddresses(req.reducersAddresses())) {
shuffleJob.startSending("external",
- new IgniteInClosure2X<HadoopProcessDescriptor, HadoopShuffleMessage>() {
- @Override public void applyx(HadoopProcessDescriptor dest,
- HadoopShuffleMessage msg) throws IgniteCheckedException {
+ new IgniteInClosure2X<HadoopProcessDescriptor, HadoopMessage>() {
+ @Override public void applyx(HadoopProcessDescriptor dest, HadoopMessage msg)
+ throws IgniteCheckedException {
comm.sendMessage(dest, msg);
}
});
[44/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/community/ignite-1.7.4' into ignite-1.8.2
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/community/ignite-1.7.4' into ignite-1.8.2
# Conflicts:
# modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
# modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/datasource/DataSource.java
# modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcRequestHandler.java
# modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
# modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
# modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
# modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/09c634e2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/09c634e2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/09c634e2
Branch: refs/heads/master
Commit: 09c634e28200e0dd94c3c2e2271b5cdf60304888
Parents: 64247b9
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 19 15:55:02 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 19 15:55:02 2016 +0300
----------------------------------------------------------------------
.../cache/store/cassandra/CassandraCacheStore.java | 3 +++
.../org/apache/ignite/internal/IgniteKernal.java | 16 ----------------
2 files changed, 3 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/09c634e2/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
----------------------------------------------------------------------
diff --git a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
index b4bed0d..70d798b 100644
--- a/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
+++ b/modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/CassandraCacheStore.java
@@ -43,6 +43,9 @@ import org.apache.ignite.cache.store.cassandra.session.CassandraSession;
import org.apache.ignite.cache.store.cassandra.session.ExecutionAssistant;
import org.apache.ignite.cache.store.cassandra.session.GenericBatchExecutionAssistant;
import org.apache.ignite.cache.store.cassandra.session.LoadCacheCustomQueryWorker;
+import org.apache.ignite.cache.store.cassandra.session.transaction.DeleteMutation;
+import org.apache.ignite.cache.store.cassandra.session.transaction.Mutation;
+import org.apache.ignite.cache.store.cassandra.session.transaction.WriteMutation;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiInClosure;
http://git-wip-us.apache.org/repos/asf/ignite/blob/09c634e2/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index e4a6115..4972d1f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2877,22 +2877,6 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
}
- /**
- * @param cacheNames Collection of cache names.
- * @param checkThreadTx If {@code true} checks that current thread does not have active transactions.
- * @return Ignite future.
- */
- public IgniteInternalFuture<?> destroyCachesAsync(Collection<String> cacheNames, boolean checkThreadTx) {
- guard();
-
- try {
- return ctx.cache().dynamicDestroyCaches(cacheNames, checkThreadTx);
- }
- finally {
- unguard();
- }
- }
-
/** {@inheritDoc} */
@Override public <K, V> IgniteCache<K, V> getOrCreateCache(String cacheName) {
guard();
[09/50] [abbrv] ignite git commit: IGNITE-4264: Fixed cluster wide
cache metrics. This closes #1336.
Posted by sb...@apache.org.
IGNITE-4264: Fixed cluster wide cache metrics. This closes #1336.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f087b661
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f087b661
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f087b661
Branch: refs/heads/master
Commit: f087b661c466c9fd27d049efa8e55a2d7bdf9834
Parents: 6fd8bf6
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Fri Dec 9 17:45:33 2016 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Fri Dec 9 17:45:33 2016 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheAdapter.java | 2 +-
...heapCacheMetricsForClusterGroupSelfTest.java | 141 +++++
.../CacheOffHeapAndSwapMetricsSelfTest.java | 621 -------------------
...LocalCacheOffHeapAndSwapMetricsSelfTest.java | 621 +++++++++++++++++++
.../IgniteCacheMetricsSelfTestSuite.java | 6 +-
5 files changed, 767 insertions(+), 624 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f087b661/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2e24e67..b30ec70 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3234,7 +3234,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** {@inheritDoc} */
@Override public CacheMetrics clusterMetrics() {
- return clusterMetrics(ctx.grid().cluster().forCacheNodes(ctx.name()));
+ return clusterMetrics(ctx.grid().cluster().forDataNodes(ctx.name()));
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f087b661/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java
new file mode 100644
index 0000000..53a20fc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/OffheapCacheMetricsForClusterGroupSelfTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.util.concurrent.CountDownLatch;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
+
+/**
+ * Test for cluster wide offheap cache metrics.
+ */
+public class OffheapCacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest {
+ /** Grid count. */
+ private static final int GRID_CNT = 3;
+
+ /** Client count */
+ private static final int CLIENT_CNT = 3;
+
+ /** Grid client mode */
+ private boolean clientMode;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setClientMode(clientMode);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ // start grids
+ clientMode = false;
+ for (int i = 0; i < GRID_CNT; i++)
+ startGrid("server-" + i);
+
+ // start clients
+ clientMode = true;
+ for (int i = 0; i < CLIENT_CNT; i++)
+ startGrid("client-" + i);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ public void testGetOffHeapPrimaryEntriesCount() throws Exception {
+ String cacheName = "testGetOffHeapPrimaryEntriesCount";
+ IgniteCache<Integer, Integer> cache = grid("client-0").createCache(cacheConfiguration(cacheName));
+
+ for (int i = 0; i < 1000; i++)
+ cache.put(i, i);
+
+ awaitMetricsUpdate();
+
+ assertGetOffHeapPrimaryEntriesCount(cacheName, 1000);
+
+ for (int j = 0; j < 1000; j++)
+ cache.get(j);
+
+ awaitMetricsUpdate();
+
+ assertGetOffHeapPrimaryEntriesCount(cacheName, 1000);
+
+ cache = grid("client-1").cache(cacheName);
+
+ for (int j = 0; j < 1000; j++)
+ cache.get(j);
+
+ awaitMetricsUpdate();
+
+ assertGetOffHeapPrimaryEntriesCount(cacheName, 1000);
+ }
+
+ /**
+ * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be receieved.
+ */
+ private void awaitMetricsUpdate() throws InterruptedException {
+ final CountDownLatch latch = new CountDownLatch((GRID_CNT + 1) * 2);
+
+ IgnitePredicate<Event> lsnr = new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event ignore) {
+ latch.countDown();
+
+ return true;
+ }
+ };
+
+ for (int i = 0; i < GRID_CNT; i++)
+ grid("server-" + i).events().localListen(lsnr, EVT_NODE_METRICS_UPDATED);
+
+ latch.await();
+ }
+
+ private void assertGetOffHeapPrimaryEntriesCount(String cacheName, int count) throws Exception {
+ for (int i = 0; i < GRID_CNT; i++) {
+ IgniteCache<Integer, Integer> cache = grid("server-" + i).cache(cacheName);
+ assertEquals(count, cache.metrics().getOffHeapPrimaryEntriesCount());
+ }
+
+ for (int i = 0; i < CLIENT_CNT; i++) {
+ IgniteCache<Integer, Integer> cache = grid("client-" + i).cache(cacheName);
+ assertEquals(count, cache.metrics().getOffHeapPrimaryEntriesCount());
+ }
+ }
+
+ private static CacheConfiguration<Integer, Integer> cacheConfiguration(String cacheName) {
+ CacheConfiguration<Integer, Integer> cfg = new CacheConfiguration<>(cacheName);
+
+ cfg.setBackups(1);
+ cfg.setStatisticsEnabled(true);
+ cfg.setMemoryMode(CacheMemoryMode.OFFHEAP_TIERED);
+ cfg.setOffHeapMaxMemory(1024 * 1024 * 1024);
+ return cfg;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f087b661/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java
deleted file mode 100644
index 6837a27..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/CacheOffHeapAndSwapMetricsSelfTest.java
+++ /dev/null
@@ -1,621 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.local;
-
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMemoryMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class CacheOffHeapAndSwapMetricsSelfTest extends GridCommonAbstractTest {
- /** Grid count. */
- private static final int GRID_CNT = 1;
-
- /** Keys count. */
- private static final int KEYS_CNT = 1000;
-
- /** Max size. */
- private static final int MAX_SIZE = 100;
-
- /** Entry size. */
- private static final int ENTRY_SIZE = 86; // Calculated as allocated size divided on entries count.
-
- /** Offheap max count. */
- private static final int OFFHEAP_MAX_CNT = KEYS_CNT / 2;
-
- /** Offheap max size. */
- private static final int OFFHEAP_MAX_SIZE = ENTRY_SIZE * OFFHEAP_MAX_CNT;
-
- /** Cache. */
- private IgniteCache<Integer, Integer> cache;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setSwapSpaceSpi(new FileSwapSpaceSpi());
-
- return cfg;
- }
-
- /**
- * @param memoryMode Cache memory mode.
- * @param offHeapSize Max off-heap size.
- * @param swapEnabled Swap enabled.
- */
- private void createCache(CacheMemoryMode memoryMode, int offHeapSize, boolean swapEnabled) {
- CacheConfiguration ccfg = defaultCacheConfiguration();
-
- ccfg.setStatisticsEnabled(true);
-
- ccfg.setCacheMode(CacheMode.LOCAL);
- ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg.setMemoryMode(memoryMode);
-
- ccfg.setOffHeapMaxMemory(offHeapSize);
- ccfg.setSwapEnabled(swapEnabled);
-
- ccfg.setEvictionPolicy(new FifoEvictionPolicy(MAX_SIZE));
-
- cache = grid(0).getOrCreateCache(ccfg);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- startGrids(GRID_CNT);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- stopAllGrids();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- if (cache != null)
- cache.destroy();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testOffHeapMetrics() throws Exception {
- createCache(CacheMemoryMode.ONHEAP_TIERED, 0, false);
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.put(i, i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapGets());
- assertEquals(0, cache.localMetrics().getOffHeapHits());
- assertEquals(0f, cache.localMetrics().getOffHeapHitPercentage());
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapMisses());
- assertEquals(100f, cache.localMetrics().getOffHeapMissPercentage());
- assertEquals(0, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(0, cache.localMetrics().getOffHeapEvictions());
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapGets());
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapHits());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapMisses());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(0, cache.localMetrics().getOffHeapEvictions());
- assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 3, cache.localMetrics().getOffHeapGets());
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapHits());
- assertEquals(100 / 3.0, cache.localMetrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapMisses());
- assertEquals(100 - (100 / 3.0), cache.localMetrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(0, cache.localMetrics().getOffHeapEvictions());
- assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.remove(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 4 - MAX_SIZE, cache.localMetrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getOffHeapHits());
- assertEquals(100 * (KEYS_CNT * 2.0 - MAX_SIZE) / (KEYS_CNT * 4.0 - MAX_SIZE),
- cache.localMetrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapMisses());
- assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - MAX_SIZE),
- cache.localMetrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(0, cache.localMetrics().getOffHeapEvictions());
- assertEquals(0, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testSwapMetrics() throws Exception {
- createCache(CacheMemoryMode.ONHEAP_TIERED, -1, true);
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.put(i, i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapGets());
- assertEquals(0, cache.localMetrics().getSwapHits());
- assertEquals(0f, cache.localMetrics().getSwapHitPercentage());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
- assertEquals(100f, cache.localMetrics().getSwapMissPercentage());
- assertEquals(0, cache.localMetrics().getSwapRemovals());
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapGets());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
-
- assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getSwapEntriesCount());
-
- for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT * 3, cache.localMetrics().getSwapGets());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
- assertEquals(100 / 3.0, cache.localMetrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
- assertEquals(100 - (100 / 3.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
-
- assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getSwapEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.remove(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT * 4 - MAX_SIZE, cache.localMetrics().getSwapGets());
- assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getSwapHits());
- assertEquals(100 * (KEYS_CNT * 2.0 - MAX_SIZE) / (KEYS_CNT * 4.0 - MAX_SIZE),
- cache.localMetrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
- assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - MAX_SIZE),
- cache.localMetrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getSwapRemovals());
-
- assertEquals(0, cache.localMetrics().getSwapEntriesCount());
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testOffHeapAndSwapMetrics() throws Exception {
- createCache(CacheMemoryMode.ONHEAP_TIERED, OFFHEAP_MAX_SIZE, true);
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.put(i, i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapGets());
- assertEquals(0, cache.localMetrics().getOffHeapHits());
- assertEquals(0f, cache.localMetrics().getOffHeapHitPercentage());
- assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapMisses());
- assertEquals(100f, cache.localMetrics().getOffHeapMissPercentage());
- assertEquals(0, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(KEYS_CNT - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapGets());
- assertEquals(0, cache.localMetrics().getSwapHits());
- assertEquals(0f, cache.localMetrics().getSwapHitPercentage());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
- assertEquals(100f, cache.localMetrics().getSwapMissPercentage());
- assertEquals(0, cache.localMetrics().getSwapRemovals());
-
- assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapGets());
- assertEquals(0, cache.localMetrics().getOffHeapHits());
- assertEquals(0.0, cache.localMetrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapMisses());
- assertEquals(100.0, cache.localMetrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(0, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(cache.localMetrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapGets());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
-
- assertEquals(KEYS_CNT - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapEntriesCount());
-
- for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 3, cache.localMetrics().getOffHeapGets());
- assertEquals(0, cache.localMetrics().getOffHeapHits());
- assertEquals(0.0, cache.localMetrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 3, cache.localMetrics().getOffHeapMisses());
- assertEquals(100.0, cache.localMetrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(0, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(cache.localMetrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT * 3, cache.localMetrics().getSwapGets());
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
- assertEquals(100 / 3.0, cache.localMetrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
- assertEquals(100 - (100 / 3.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
-
- assertEquals(KEYS_CNT - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.remove(i);
-
- printStat();
-
- assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 4 - MAX_SIZE, cache.localMetrics().getOffHeapGets());
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapHits());
- assertEquals(100 * OFFHEAP_MAX_CNT / (KEYS_CNT * 4.0 - MAX_SIZE),
- cache.localMetrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 4 - OFFHEAP_MAX_CNT - MAX_SIZE, cache.localMetrics().getOffHeapMisses());
- assertEquals(100 * (KEYS_CNT * 4 - OFFHEAP_MAX_CNT - MAX_SIZE) / (KEYS_CNT * 4.0 - MAX_SIZE),
- cache.localMetrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapRemovals());
-
- assertEquals(cache.localMetrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
- assertEquals(0, cache.localMetrics().getOffHeapEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
- assertEquals(KEYS_CNT * 4 - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapGets());
- assertEquals(KEYS_CNT * 2 - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapHits());
- assertEquals(100 * (KEYS_CNT * 2.0 - MAX_SIZE - OFFHEAP_MAX_CNT) / (KEYS_CNT * 4.0 - MAX_SIZE - OFFHEAP_MAX_CNT),
- cache.localMetrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
- assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - MAX_SIZE - OFFHEAP_MAX_CNT),
- cache.localMetrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2 - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapRemovals());
-
- assertEquals(0, cache.localMetrics().getSwapEntriesCount());
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testOffHeapMetricsInOffHeapTiered() throws Exception {
- createCache(CacheMemoryMode.OFFHEAP_TIERED, 0, false);
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.put(i, i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapPuts());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapGets());
- assertEquals(0, cache.metrics().getOffHeapHits());
- assertEquals(0f, cache.metrics().getOffHeapHitPercentage());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapMisses());
- assertEquals(100f, cache.metrics().getOffHeapMissPercentage());
- assertEquals(0, cache.metrics().getOffHeapRemovals());
-
- assertEquals(0, cache.metrics().getOffHeapEvictions());
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapEntriesCount());
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapHits());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapMisses());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(0, cache.metrics().getOffHeapRemovals());
-
- assertEquals(0, cache.metrics().getOffHeapEvictions());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapEntriesCount());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 3, cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapHits());
- assertEquals(100 / 3.0, cache.metrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapMisses());
- assertEquals(100 - (100 / 3.0), cache.metrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(0, cache.metrics().getOffHeapRemovals());
-
- assertEquals(0, cache.metrics().getOffHeapEvictions());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapEntriesCount());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.remove(i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 4, cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapHits());
- assertEquals(100 * (KEYS_CNT * 2.0) / (KEYS_CNT * 4.0),
- cache.metrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapMisses());
- assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0),
- cache.metrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapRemovals());
-
- assertEquals(0, cache.metrics().getOffHeapEvictions());
- assertEquals(0, cache.metrics().getOffHeapEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testOffHeapAndSwapMetricsInOffHeapTiered() throws Exception {
- createCache(CacheMemoryMode.OFFHEAP_TIERED, OFFHEAP_MAX_SIZE, true);
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.put(i, i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapGets());
- assertEquals(0, cache.metrics().getOffHeapHits());
- assertEquals(0f, cache.metrics().getOffHeapHitPercentage());
- assertEquals(KEYS_CNT, cache.metrics().getOffHeapMisses());
- assertEquals(100f, cache.metrics().getOffHeapMissPercentage());
- assertEquals(0, cache.metrics().getOffHeapRemovals());
-
- assertEquals(KEYS_CNT - OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEvictions());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEntriesCount());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
- assertEquals(KEYS_CNT, cache.metrics().getSwapGets());
- assertEquals(0, cache.metrics().getSwapHits());
- assertEquals(0f, cache.metrics().getSwapHitPercentage());
- assertEquals(KEYS_CNT, cache.metrics().getSwapMisses());
- assertEquals(100f, cache.metrics().getSwapMissPercentage());
- assertEquals(0, cache.metrics().getSwapRemovals());
-
- assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapPuts());
- assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapGets());
- assertEquals(0, cache.metrics().getOffHeapHits());
- assertEquals(0.0, cache.metrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapMisses());
- assertEquals(100.0, cache.metrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(0, cache.metrics().getOffHeapRemovals());
-
- assertEquals(cache.metrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEvictions());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEntriesCount());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
- assertEquals(KEYS_CNT * 2, cache.metrics().getSwapGets());
- assertEquals(KEYS_CNT, cache.metrics().getSwapHits());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.metrics().getSwapMisses());
- assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.metrics().getSwapRemovals());
-
- assertEquals(KEYS_CNT - OFFHEAP_MAX_CNT, cache.metrics().getSwapEntriesCount());
-
- for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
- cache.get(i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 3, cache.metrics().getOffHeapGets());
- assertEquals(0, cache.metrics().getOffHeapHits());
- assertEquals(0.0, cache.metrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 3, cache.metrics().getOffHeapMisses());
- assertEquals(100.0, cache.metrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(0, cache.metrics().getOffHeapRemovals());
-
- assertEquals(cache.metrics().getCacheEvictions() - OFFHEAP_MAX_CNT - KEYS_CNT,
- cache.metrics().getOffHeapEvictions());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEntriesCount());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
- assertEquals(KEYS_CNT * 3, cache.metrics().getSwapGets());
- assertEquals(KEYS_CNT, cache.metrics().getSwapHits());
- assertEquals(100 / 3.0, cache.metrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.metrics().getSwapMisses());
- assertEquals(100 - (100 / 3.0), cache.metrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT, cache.metrics().getSwapRemovals());
-
- assertEquals(KEYS_CNT - OFFHEAP_MAX_CNT, cache.metrics().getSwapEntriesCount());
-
- for (int i = 0; i < KEYS_CNT; i++)
- cache.remove(i);
-
- printStat();
-
- assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
- assertEquals(KEYS_CNT * 4, cache.metrics().getOffHeapGets());
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapHits());
- assertEquals(100 * OFFHEAP_MAX_CNT / (KEYS_CNT * 4.0),
- cache.metrics().getOffHeapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 4 - OFFHEAP_MAX_CNT, cache.metrics().getOffHeapMisses());
- assertEquals(100 * (KEYS_CNT * 4 - OFFHEAP_MAX_CNT) / (KEYS_CNT * 4.0),
- cache.metrics().getOffHeapMissPercentage(), 0.1);
- assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapRemovals());
-
- assertEquals(cache.metrics().getCacheEvictions() - OFFHEAP_MAX_CNT - 2 * KEYS_CNT, cache.metrics().getOffHeapEvictions());
- assertEquals(0, cache.metrics().getOffHeapEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapPrimaryEntriesCount());
- assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
-
- assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
- assertEquals(KEYS_CNT * 4 - OFFHEAP_MAX_CNT, cache.metrics().getSwapGets());
- assertEquals(KEYS_CNT * 2 - OFFHEAP_MAX_CNT, cache.metrics().getSwapHits());
- assertEquals(100 * (KEYS_CNT * 2.0 - OFFHEAP_MAX_CNT) / (KEYS_CNT * 4.0 - OFFHEAP_MAX_CNT),
- cache.metrics().getSwapHitPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2, cache.metrics().getSwapMisses());
- assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - OFFHEAP_MAX_CNT),
- cache.metrics().getSwapMissPercentage(), 0.1);
- assertEquals(KEYS_CNT * 2 - OFFHEAP_MAX_CNT, cache.metrics().getSwapRemovals());
-
- assertEquals(0, cache.metrics().getSwapEntriesCount());
- }
-
- /**
- * Prints stats.
- */
- protected void printStat() {
- System.out.println("!!! -------------------------------------------------------");
- System.out.println("!!! Puts: cache = " + cache.localMetrics().getCachePuts() +
- ", offheap = " + cache.localMetrics().getOffHeapPuts() +
- ", swap = " + cache.localMetrics().getSwapPuts());
- System.out.println("!!! Gets: cache = " + cache.localMetrics().getCacheGets() +
- ", offheap = " + cache.localMetrics().getOffHeapGets() +
- ", swap = " + cache.localMetrics().getSwapGets());
- System.out.println("!!! Removes: cache = " + cache.localMetrics().getCacheRemovals() +
- ", offheap = " + cache.localMetrics().getOffHeapRemovals() +
- ", swap = " + cache.localMetrics().getSwapRemovals());
- System.out.println("!!! Evictions: cache = " + cache.localMetrics().getCacheEvictions() +
- ", offheap = " + cache.localMetrics().getOffHeapEvictions() +
- ", swap = none" );
- System.out.println("!!! Hits: cache = " + cache.localMetrics().getCacheHits() +
- ", offheap = " + cache.localMetrics().getOffHeapHits() +
- ", swap = " + cache.localMetrics().getSwapHits());
- System.out.println("!!! Hit(%): cache = " + cache.localMetrics().getCacheHitPercentage() +
- ", offheap = " + cache.localMetrics().getOffHeapHitPercentage() +
- ", swap = " + cache.localMetrics().getSwapHitPercentage());
- System.out.println("!!! Misses: cache = " + cache.localMetrics().getCacheMisses() +
- ", offheap = " + cache.localMetrics().getOffHeapMisses() +
- ", swap = " + cache.localMetrics().getSwapMisses());
- System.out.println("!!! Miss(%): cache = " + cache.localMetrics().getCacheMissPercentage() +
- ", offheap = " + cache.localMetrics().getOffHeapMissPercentage() +
- ", swap = " + cache.localMetrics().getSwapMissPercentage());
- System.out.println("!!! Entries: cache = " + cache.localMetrics().getSize() +
- ", offheap = " + cache.localMetrics().getOffHeapEntriesCount() +
- ", swap = " + cache.localMetrics().getSwapEntriesCount());
- System.out.println("!!! Size: cache = none" +
- ", offheap = " + cache.localMetrics().getOffHeapAllocatedSize() +
- ", swap = " + cache.localMetrics().getSwapSize());
- System.out.println();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f087b661/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java
new file mode 100644
index 0000000..ae8807f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/LocalCacheOffHeapAndSwapMetricsSelfTest.java
@@ -0,0 +1,621 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.local;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMemoryMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.swapspace.file.FileSwapSpaceSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class LocalCacheOffHeapAndSwapMetricsSelfTest extends GridCommonAbstractTest {
+ /** Grid count. */
+ private static final int GRID_CNT = 1;
+
+ /** Keys count. */
+ private static final int KEYS_CNT = 1000;
+
+ /** Max size. */
+ private static final int MAX_SIZE = 100;
+
+ /** Entry size. */
+ private static final int ENTRY_SIZE = 86; // Calculated as allocated size divided on entries count.
+
+ /** Offheap max count. */
+ private static final int OFFHEAP_MAX_CNT = KEYS_CNT / 2;
+
+ /** Offheap max size. */
+ private static final int OFFHEAP_MAX_SIZE = ENTRY_SIZE * OFFHEAP_MAX_CNT;
+
+ /** Cache. */
+ private IgniteCache<Integer, Integer> cache;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setSwapSpaceSpi(new FileSwapSpaceSpi());
+
+ return cfg;
+ }
+
+ /**
+ * @param memoryMode Cache memory mode.
+ * @param offHeapSize Max off-heap size.
+ * @param swapEnabled Swap enabled.
+ */
+ private void createCache(CacheMemoryMode memoryMode, int offHeapSize, boolean swapEnabled) {
+ CacheConfiguration ccfg = defaultCacheConfiguration();
+
+ ccfg.setStatisticsEnabled(true);
+
+ ccfg.setCacheMode(CacheMode.LOCAL);
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setMemoryMode(memoryMode);
+
+ ccfg.setOffHeapMaxMemory(offHeapSize);
+ ccfg.setSwapEnabled(swapEnabled);
+
+ ccfg.setEvictionPolicy(new FifoEvictionPolicy(MAX_SIZE));
+
+ cache = grid(0).getOrCreateCache(ccfg);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGrids(GRID_CNT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ if (cache != null)
+ cache.destroy();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testOffHeapMetrics() throws Exception {
+ createCache(CacheMemoryMode.ONHEAP_TIERED, 0, false);
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.put(i, i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapGets());
+ assertEquals(0, cache.localMetrics().getOffHeapHits());
+ assertEquals(0f, cache.localMetrics().getOffHeapHitPercentage());
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100f, cache.localMetrics().getOffHeapMissPercentage());
+ assertEquals(0, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapGets());
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapHits());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 3, cache.localMetrics().getOffHeapGets());
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapHits());
+ assertEquals(100 / 3.0, cache.localMetrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100 - (100 / 3.0), cache.localMetrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.remove(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 4 - MAX_SIZE, cache.localMetrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getOffHeapHits());
+ assertEquals(100 * (KEYS_CNT * 2.0 - MAX_SIZE) / (KEYS_CNT * 4.0 - MAX_SIZE),
+ cache.localMetrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - MAX_SIZE),
+ cache.localMetrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(0, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testSwapMetrics() throws Exception {
+ createCache(CacheMemoryMode.ONHEAP_TIERED, -1, true);
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.put(i, i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapGets());
+ assertEquals(0, cache.localMetrics().getSwapHits());
+ assertEquals(0f, cache.localMetrics().getSwapHitPercentage());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
+ assertEquals(100f, cache.localMetrics().getSwapMissPercentage());
+ assertEquals(0, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapGets());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getSwapEntriesCount());
+
+ for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 3, cache.localMetrics().getSwapGets());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
+ assertEquals(100 / 3.0, cache.localMetrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
+ assertEquals(100 - (100 / 3.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(KEYS_CNT - MAX_SIZE, cache.localMetrics().getSwapEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.remove(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 4 - MAX_SIZE, cache.localMetrics().getSwapGets());
+ assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getSwapHits());
+ assertEquals(100 * (KEYS_CNT * 2.0 - MAX_SIZE) / (KEYS_CNT * 4.0 - MAX_SIZE),
+ cache.localMetrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
+ assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - MAX_SIZE),
+ cache.localMetrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2 - MAX_SIZE, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(0, cache.localMetrics().getSwapEntriesCount());
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testOffHeapAndSwapMetrics() throws Exception {
+ createCache(CacheMemoryMode.ONHEAP_TIERED, OFFHEAP_MAX_SIZE, true);
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.put(i, i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapGets());
+ assertEquals(0, cache.localMetrics().getOffHeapHits());
+ assertEquals(0f, cache.localMetrics().getOffHeapHitPercentage());
+ assertEquals(KEYS_CNT, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100f, cache.localMetrics().getOffHeapMissPercentage());
+ assertEquals(0, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(KEYS_CNT - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapGets());
+ assertEquals(0, cache.localMetrics().getSwapHits());
+ assertEquals(0f, cache.localMetrics().getSwapHitPercentage());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
+ assertEquals(100f, cache.localMetrics().getSwapMissPercentage());
+ assertEquals(0, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapGets());
+ assertEquals(0, cache.localMetrics().getOffHeapHits());
+ assertEquals(0.0, cache.localMetrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100.0, cache.localMetrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(0, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(cache.localMetrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapGets());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapMisses());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(KEYS_CNT - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapEntriesCount());
+
+ for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 3, cache.localMetrics().getOffHeapGets());
+ assertEquals(0, cache.localMetrics().getOffHeapHits());
+ assertEquals(0.0, cache.localMetrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 3, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100.0, cache.localMetrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(0, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(cache.localMetrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 3, cache.localMetrics().getSwapGets());
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapHits());
+ assertEquals(100 / 3.0, cache.localMetrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
+ assertEquals(100 - (100 / 3.0), cache.localMetrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(KEYS_CNT - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.remove(i);
+
+ printStat();
+
+ assertEquals(cache.localMetrics().getCacheEvictions(), cache.localMetrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 4 - MAX_SIZE, cache.localMetrics().getOffHeapGets());
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapHits());
+ assertEquals(100 * OFFHEAP_MAX_CNT / (KEYS_CNT * 4.0 - MAX_SIZE),
+ cache.localMetrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 4 - OFFHEAP_MAX_CNT - MAX_SIZE, cache.localMetrics().getOffHeapMisses());
+ assertEquals(100 * (KEYS_CNT * 4 - OFFHEAP_MAX_CNT - MAX_SIZE) / (KEYS_CNT * 4.0 - MAX_SIZE),
+ cache.localMetrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapRemovals());
+
+ assertEquals(cache.localMetrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.localMetrics().getOffHeapEvictions());
+ assertEquals(0, cache.localMetrics().getOffHeapEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.localMetrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.localMetrics().getOffHeapEvictions(), cache.localMetrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 4 - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapGets());
+ assertEquals(KEYS_CNT * 2 - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapHits());
+ assertEquals(100 * (KEYS_CNT * 2.0 - MAX_SIZE - OFFHEAP_MAX_CNT) / (KEYS_CNT * 4.0 - MAX_SIZE - OFFHEAP_MAX_CNT),
+ cache.localMetrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.localMetrics().getSwapMisses());
+ assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - MAX_SIZE - OFFHEAP_MAX_CNT),
+ cache.localMetrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2 - MAX_SIZE - OFFHEAP_MAX_CNT, cache.localMetrics().getSwapRemovals());
+
+ assertEquals(0, cache.localMetrics().getSwapEntriesCount());
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testOffHeapMetricsInOffHeapTiered() throws Exception {
+ createCache(CacheMemoryMode.OFFHEAP_TIERED, 0, false);
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.put(i, i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapGets());
+ assertEquals(0, cache.metrics().getOffHeapHits());
+ assertEquals(0f, cache.metrics().getOffHeapHitPercentage());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapMisses());
+ assertEquals(100f, cache.metrics().getOffHeapMissPercentage());
+ assertEquals(0, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.metrics().getOffHeapEvictions());
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapEntriesCount());
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapHits());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapMisses());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(0, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.metrics().getOffHeapEvictions());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 3, cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapHits());
+ assertEquals(100 / 3.0, cache.metrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapMisses());
+ assertEquals(100 - (100 / 3.0), cache.metrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(0, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.metrics().getOffHeapEvictions());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.remove(i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 4, cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapHits());
+ assertEquals(100 * (KEYS_CNT * 2.0) / (KEYS_CNT * 4.0),
+ cache.metrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapMisses());
+ assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0),
+ cache.metrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(0, cache.metrics().getOffHeapEvictions());
+ assertEquals(0, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testOffHeapAndSwapMetricsInOffHeapTiered() throws Exception {
+ createCache(CacheMemoryMode.OFFHEAP_TIERED, OFFHEAP_MAX_SIZE, true);
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.put(i, i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapGets());
+ assertEquals(0, cache.metrics().getOffHeapHits());
+ assertEquals(0f, cache.metrics().getOffHeapHitPercentage());
+ assertEquals(KEYS_CNT, cache.metrics().getOffHeapMisses());
+ assertEquals(100f, cache.metrics().getOffHeapMissPercentage());
+ assertEquals(0, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(KEYS_CNT - OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEvictions());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
+ assertEquals(KEYS_CNT, cache.metrics().getSwapGets());
+ assertEquals(0, cache.metrics().getSwapHits());
+ assertEquals(0f, cache.metrics().getSwapHitPercentage());
+ assertEquals(KEYS_CNT, cache.metrics().getSwapMisses());
+ assertEquals(100f, cache.metrics().getSwapMissPercentage());
+ assertEquals(0, cache.metrics().getSwapRemovals());
+
+ assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapPuts());
+ assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapGets());
+ assertEquals(0, cache.metrics().getOffHeapHits());
+ assertEquals(0.0, cache.metrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.metrics().getOffHeapMisses());
+ assertEquals(100.0, cache.metrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(0, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(cache.metrics().getCacheEvictions() - OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEvictions());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 2, cache.metrics().getSwapGets());
+ assertEquals(KEYS_CNT, cache.metrics().getSwapHits());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.metrics().getSwapMisses());
+ assertEquals(100 * KEYS_CNT / (KEYS_CNT * 2.0), cache.metrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.metrics().getSwapRemovals());
+
+ assertEquals(KEYS_CNT - OFFHEAP_MAX_CNT, cache.metrics().getSwapEntriesCount());
+
+ for (int i = KEYS_CNT; i < KEYS_CNT * 2; i++)
+ cache.get(i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 3, cache.metrics().getOffHeapGets());
+ assertEquals(0, cache.metrics().getOffHeapHits());
+ assertEquals(0.0, cache.metrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 3, cache.metrics().getOffHeapMisses());
+ assertEquals(100.0, cache.metrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(0, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(cache.metrics().getCacheEvictions() - OFFHEAP_MAX_CNT - KEYS_CNT,
+ cache.metrics().getOffHeapEvictions());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 3, cache.metrics().getSwapGets());
+ assertEquals(KEYS_CNT, cache.metrics().getSwapHits());
+ assertEquals(100 / 3.0, cache.metrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.metrics().getSwapMisses());
+ assertEquals(100 - (100 / 3.0), cache.metrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT, cache.metrics().getSwapRemovals());
+
+ assertEquals(KEYS_CNT - OFFHEAP_MAX_CNT, cache.metrics().getSwapEntriesCount());
+
+ for (int i = 0; i < KEYS_CNT; i++)
+ cache.remove(i);
+
+ printStat();
+
+ assertEquals(cache.metrics().getCacheEvictions(), cache.metrics().getOffHeapGets());
+ assertEquals(KEYS_CNT * 4, cache.metrics().getOffHeapGets());
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapHits());
+ assertEquals(100 * OFFHEAP_MAX_CNT / (KEYS_CNT * 4.0),
+ cache.metrics().getOffHeapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 4 - OFFHEAP_MAX_CNT, cache.metrics().getOffHeapMisses());
+ assertEquals(100 * (KEYS_CNT * 4 - OFFHEAP_MAX_CNT) / (KEYS_CNT * 4.0),
+ cache.metrics().getOffHeapMissPercentage(), 0.1);
+ assertEquals(OFFHEAP_MAX_CNT, cache.metrics().getOffHeapRemovals());
+
+ assertEquals(cache.metrics().getCacheEvictions() - OFFHEAP_MAX_CNT - 2 * KEYS_CNT, cache.metrics().getOffHeapEvictions());
+ assertEquals(0, cache.metrics().getOffHeapEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapPrimaryEntriesCount());
+ assertEquals(0, cache.metrics().getOffHeapBackupEntriesCount());
+
+ assertEquals(cache.metrics().getOffHeapEvictions(), cache.metrics().getSwapPuts());
+ assertEquals(KEYS_CNT * 4 - OFFHEAP_MAX_CNT, cache.metrics().getSwapGets());
+ assertEquals(KEYS_CNT * 2 - OFFHEAP_MAX_CNT, cache.metrics().getSwapHits());
+ assertEquals(100 * (KEYS_CNT * 2.0 - OFFHEAP_MAX_CNT) / (KEYS_CNT * 4.0 - OFFHEAP_MAX_CNT),
+ cache.metrics().getSwapHitPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2, cache.metrics().getSwapMisses());
+ assertEquals(100 * KEYS_CNT * 2.0 / (KEYS_CNT * 4.0 - OFFHEAP_MAX_CNT),
+ cache.metrics().getSwapMissPercentage(), 0.1);
+ assertEquals(KEYS_CNT * 2 - OFFHEAP_MAX_CNT, cache.metrics().getSwapRemovals());
+
+ assertEquals(0, cache.metrics().getSwapEntriesCount());
+ }
+
+ /**
+ * Prints stats.
+ */
+ protected void printStat() {
+ System.out.println("!!! -------------------------------------------------------");
+ System.out.println("!!! Puts: cache = " + cache.localMetrics().getCachePuts() +
+ ", offheap = " + cache.localMetrics().getOffHeapPuts() +
+ ", swap = " + cache.localMetrics().getSwapPuts());
+ System.out.println("!!! Gets: cache = " + cache.localMetrics().getCacheGets() +
+ ", offheap = " + cache.localMetrics().getOffHeapGets() +
+ ", swap = " + cache.localMetrics().getSwapGets());
+ System.out.println("!!! Removes: cache = " + cache.localMetrics().getCacheRemovals() +
+ ", offheap = " + cache.localMetrics().getOffHeapRemovals() +
+ ", swap = " + cache.localMetrics().getSwapRemovals());
+ System.out.println("!!! Evictions: cache = " + cache.localMetrics().getCacheEvictions() +
+ ", offheap = " + cache.localMetrics().getOffHeapEvictions() +
+ ", swap = none" );
+ System.out.println("!!! Hits: cache = " + cache.localMetrics().getCacheHits() +
+ ", offheap = " + cache.localMetrics().getOffHeapHits() +
+ ", swap = " + cache.localMetrics().getSwapHits());
+ System.out.println("!!! Hit(%): cache = " + cache.localMetrics().getCacheHitPercentage() +
+ ", offheap = " + cache.localMetrics().getOffHeapHitPercentage() +
+ ", swap = " + cache.localMetrics().getSwapHitPercentage());
+ System.out.println("!!! Misses: cache = " + cache.localMetrics().getCacheMisses() +
+ ", offheap = " + cache.localMetrics().getOffHeapMisses() +
+ ", swap = " + cache.localMetrics().getSwapMisses());
+ System.out.println("!!! Miss(%): cache = " + cache.localMetrics().getCacheMissPercentage() +
+ ", offheap = " + cache.localMetrics().getOffHeapMissPercentage() +
+ ", swap = " + cache.localMetrics().getSwapMissPercentage());
+ System.out.println("!!! Entries: cache = " + cache.localMetrics().getSize() +
+ ", offheap = " + cache.localMetrics().getOffHeapEntriesCount() +
+ ", swap = " + cache.localMetrics().getSwapEntriesCount());
+ System.out.println("!!! Size: cache = none" +
+ ", offheap = " + cache.localMetrics().getOffHeapAllocatedSize() +
+ ", swap = " + cache.localMetrics().getSwapSize());
+ System.out.println();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f087b661/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index 90ecbda..f3dd1c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.cache.CacheMetricsForClusterGroupSelfTest;
+import org.apache.ignite.internal.processors.cache.OffheapCacheMetricsForClusterGroupSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicPartitionedTckMetricsSelfTestImpl;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearMetricsSelfTest;
@@ -26,7 +27,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePar
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheAtomicReplicatedMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMetricsSelfTest;
-import org.apache.ignite.internal.processors.cache.local.CacheOffHeapAndSwapMetricsSelfTest;
+import org.apache.ignite.internal.processors.cache.local.LocalCacheOffHeapAndSwapMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalMetricsNoStoreSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalMetricsSelfTest;
import org.apache.ignite.internal.processors.cache.local.GridCacheAtomicLocalTckMetricsSelfTestImpl;
@@ -48,7 +49,7 @@ public class IgniteCacheMetricsSelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheReplicatedMetricsSelfTest.class);
suite.addTestSuite(GridCachePartitionedMetricsSelfTest.class);
suite.addTestSuite(GridCachePartitionedHitsAndMissesSelfTest.class);
- suite.addTestSuite(CacheOffHeapAndSwapMetricsSelfTest.class);
+ suite.addTestSuite(LocalCacheOffHeapAndSwapMetricsSelfTest.class);
// Atomic cache.
suite.addTestSuite(GridCacheAtomicLocalMetricsSelfTest.class);
@@ -60,6 +61,7 @@ public class IgniteCacheMetricsSelfTestSuite extends TestSuite {
// Cluster wide metrics.
suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class);
+ suite.addTestSuite(OffheapCacheMetricsForClusterGroupSelfTest.class);
return suite;
}
[35/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index c8e2e0b..bc1f173 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -37,6 +37,7 @@ import java.nio.channels.WritableByteChannel;
import java.nio.channels.spi.SelectorProvider;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Deque;
import java.util.Iterator;
import java.util.List;
@@ -44,6 +45,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
@@ -63,6 +66,7 @@ import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageReader;
import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -86,11 +90,14 @@ import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.NIO_OPER
*
*/
public class GridNioServer<T> {
+ /** */
+ public static final String IGNITE_IO_BALANCE_RANDOM_BALANCE = "IGNITE_IO_BALANCE_RANDOM_BALANCER";
+
/** Default session write timeout. */
public static final int DFLT_SES_WRITE_TIMEOUT = 5000;
/** Default send queue limit. */
- public static final int DFLT_SEND_QUEUE_LIMIT = 1024;
+ public static final int DFLT_SEND_QUEUE_LIMIT = 0;
/** Time, which server will wait before retry operation. */
private static final long ERR_WAIT_TIME = 2000;
@@ -122,6 +129,9 @@ public class GridNioServer<T> {
}
}
+ /** Defines how many times selector should do {@code selectNow()} before doing {@code select(long)}. */
+ private long selectorSpins;
+
/** Accept worker thread. */
@GridToStringExclude
private final IgniteThread acceptThread;
@@ -145,9 +155,13 @@ public class GridNioServer<T> {
/** Flag indicating if this server should use direct buffers. */
private final boolean directBuf;
- /** Index to select which thread will serve next socket channel. Using round-robin balancing. */
+ /** Index to select which thread will serve next incoming socket channel. Using round-robin balancing. */
+ @GridToStringExclude
+ private int readBalanceIdx;
+
+ /** Index to select which thread will serve next out socket channel. Using round-robin balancing. */
@GridToStringExclude
- private int balanceIdx;
+ private int writeBalanceIdx = 1;
/** Tcp no delay flag. */
private final boolean tcpNoDelay;
@@ -204,12 +218,25 @@ public class GridNioServer<T> {
/** Optional listener to monitor outbound message queue size. */
private IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr;
+ /** */
+ private final AtomicLong readerMoveCnt = new AtomicLong();
+
+ /** */
+ private final AtomicLong writerMoveCnt = new AtomicLong();
+
+ /** */
+ private final IgniteRunnable balancer;
+
/**
* @param addr Address.
* @param port Port.
* @param log Log.
* @param selectorCnt Count of selectors and selecting threads.
* @param gridName Grid name.
+ * @param srvName Logical server name for threads identification.
+ * @param selectorSpins Defines how many non-blocking {@code selector.selectNow()} should be made before
+ * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+ * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
* @param tcpNoDelay If TCP_NODELAY option should be set to accepted sockets.
* @param directBuf Direct buffer flag.
* @param order Byte order.
@@ -223,6 +250,7 @@ public class GridNioServer<T> {
* @param writerFactory Writer factory.
* @param skipRecoveryPred Skip recovery predicate.
* @param msgQueueLsnr Message queue size listener.
+ * @param balancing NIO sessions balancing flag.
* @param filters Filters for this server.
* @throws IgniteCheckedException If failed.
*/
@@ -232,6 +260,8 @@ public class GridNioServer<T> {
IgniteLogger log,
int selectorCnt,
@Nullable String gridName,
+ @Nullable String srvName,
+ long selectorSpins,
boolean tcpNoDelay,
boolean directBuf,
ByteOrder order,
@@ -245,6 +275,7 @@ public class GridNioServer<T> {
GridNioMessageWriterFactory writerFactory,
IgnitePredicate<Message> skipRecoveryPred,
IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr,
+ boolean balancing,
GridNioFilter... filters
) throws IgniteCheckedException {
if (port != -1)
@@ -268,6 +299,7 @@ public class GridNioServer<T> {
this.sockSndBuf = sockSndBuf;
this.sndQueueLimit = sndQueueLimit;
this.msgQueueLsnr = msgQueueLsnr;
+ this.selectorSpins = selectorSpins;
filterChain = new GridNioFilterChain<>(log, lsnr, new HeadFilter(), filters);
@@ -299,9 +331,16 @@ public class GridNioServer<T> {
clientThreads = new IgniteThread[selectorCnt];
for (int i = 0; i < selectorCnt; i++) {
+ String threadName;
+
+ if (srvName == null)
+ threadName = "grid-nio-worker-" + i;
+ else
+ threadName = "grid-nio-worker-" + srvName + "-" + i;
+
AbstractNioClientWorker worker = directMode ?
- new DirectNioClientWorker(i, gridName, "grid-nio-worker-" + i, log) :
- new ByteBufferNioClientWorker(i, gridName, "grid-nio-worker-" + i, log);
+ new DirectNioClientWorker(i, gridName, threadName, log) :
+ new ByteBufferNioClientWorker(i, gridName, threadName, log);
clientWorkers.add(worker);
@@ -315,6 +354,32 @@ public class GridNioServer<T> {
this.writerFactory = writerFactory;
this.skipRecoveryPred = skipRecoveryPred != null ? skipRecoveryPred : F.<Message>alwaysFalse();
+
+ long balancePeriod = IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_IO_BALANCE_PERIOD, 5000);
+
+ IgniteRunnable balancer0 = null;
+
+ if (balancing && balancePeriod > 0) {
+ boolean rndBalance = IgniteSystemProperties.getBoolean(IGNITE_IO_BALANCE_RANDOM_BALANCE, false);
+
+ balancer0 = rndBalance ? new RandomBalancer() : new SizeBasedBalancer(balancePeriod);
+ }
+
+ this.balancer = balancer0;
+ }
+
+ /**
+ * @return Number of reader sessions move.
+ */
+ public long readerMoveCount() {
+ return readerMoveCnt.get();
+ }
+
+ /**
+ * @return Number of reader writer move.
+ */
+ public long writerMoveCount() {
+ return writerMoveCnt.get();
}
/**
@@ -377,6 +442,13 @@ public class GridNioServer<T> {
}
/**
+ * @return Selector spins.
+ */
+ public long selectorSpins() {
+ return selectorSpins;
+ }
+
+ /**
* @param ses Session to close.
* @return Future for operation.
*/
@@ -390,7 +462,7 @@ public class GridNioServer<T> {
NioOperationFuture<Boolean> fut = new NioOperationFuture<>(impl, NioOperation.CLOSE);
- clientWorkers.get(impl.selectorIndex()).offer(fut);
+ impl.offerStateChange(fut);
return fut;
}
@@ -398,61 +470,91 @@ public class GridNioServer<T> {
/**
* @param ses Session.
* @param msg Message.
+ * @param createFut {@code True} if future should be created.
* @return Future for operation.
*/
- GridNioFuture<?> send(GridNioSession ses, ByteBuffer msg) {
- assert ses instanceof GridSelectorNioSessionImpl;
+ GridNioFuture<?> send(GridNioSession ses, ByteBuffer msg, boolean createFut) throws IgniteCheckedException {
+ assert ses instanceof GridSelectorNioSessionImpl : ses;
GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
- NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg);
+ if (createFut) {
+ NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg);
- send0(impl, fut, false);
+ send0(impl, fut, false);
- return fut;
+ return fut;
+ }
+ else {
+ SessionWriteRequest req = new WriteRequestImpl(ses, msg, true);
+
+ send0(impl, req, false);
+
+ return null;
+ }
}
/**
* @param ses Session.
* @param msg Message.
+ * @param createFut {@code True} if future should be created.
* @return Future for operation.
*/
- GridNioFuture<?> send(GridNioSession ses, Message msg) {
+ GridNioFuture<?> send(GridNioSession ses, Message msg, boolean createFut) throws IgniteCheckedException {
assert ses instanceof GridSelectorNioSessionImpl;
GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
- NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
- skipRecoveryPred.apply(msg));
+ if (createFut) {
+ NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
+ skipRecoveryPred.apply(msg));
- send0(impl, fut, false);
+ send0(impl, fut, false);
- return fut;
+ return fut;
+ }
+ else {
+ SessionWriteRequest req = new WriteRequestImpl(ses, msg, skipRecoveryPred.apply(msg));
+
+ send0(impl, req, false);
+
+ return null;
+ }
}
/**
* @param ses Session.
- * @param fut Future.
+ * @param req Request.
* @param sys System message flag.
+ * @throws IgniteCheckedException If session was closed.
*/
- private void send0(GridSelectorNioSessionImpl ses, NioOperationFuture<?> fut, boolean sys) {
+ private void send0(GridSelectorNioSessionImpl ses, SessionWriteRequest req, boolean sys) throws IgniteCheckedException {
assert ses != null;
- assert fut != null;
+ assert req != null;
- int msgCnt = sys ? ses.offerSystemFuture(fut) : ses.offerFuture(fut);
+ int msgCnt = sys ? ses.offerSystemFuture(req) : ses.offerFuture(req);
IgniteInClosure<IgniteException> ackC;
if (!sys && (ackC = ses.removeMeta(ACK_CLOSURE.ordinal())) != null)
- fut.ackClosure(ackC);
+ req.ackClosure(ackC);
if (ses.closed()) {
- if (ses.removeFuture(fut))
- fut.connectionClosed();
+ if (ses.removeFuture(req)) {
+ IOException err = new IOException("Failed to send message (connection was closed): " + ses);
+
+ req.onError(err);
+
+ if (!(req instanceof GridNioFuture))
+ throw new IgniteCheckedException(err);
+ }
+ }
+ else if (!ses.procWrite.get() && ses.procWrite.compareAndSet(false, true)) {
+ AbstractNioClientWorker worker = (AbstractNioClientWorker)ses.worker();
+
+ if (worker != null)
+ worker.offer((SessionChangeRequest)req);
}
- else if (msgCnt == 1)
- // Change from 0 to 1 means that worker thread should be waken up.
- clientWorkers.get(ses.selectorIndex()).offer(fut);
if (msgQueueLsnr != null)
msgQueueLsnr.apply(ses, msgCnt);
@@ -463,10 +565,10 @@ public class GridNioServer<T> {
*
* @param ses Session.
* @param msg Message.
- * @return Future.
+ * @throws IgniteCheckedException If session was closed.
*/
- public GridNioFuture<?> sendSystem(GridNioSession ses, Message msg) {
- return sendSystem(ses, msg, null);
+ public void sendSystem(GridNioSession ses, Message msg) throws IgniteCheckedException {
+ sendSystem(ses, msg, null);
}
/**
@@ -475,27 +577,30 @@ public class GridNioServer<T> {
* @param ses Session.
* @param msg Message.
* @param lsnr Future listener notified from the session thread.
- * @return Future.
+ * @throws IgniteCheckedException If session was closed.
*/
- public GridNioFuture<?> sendSystem(GridNioSession ses,
+ public void sendSystem(GridNioSession ses,
Message msg,
- @Nullable IgniteInClosure<? super IgniteInternalFuture<?>> lsnr) {
+ @Nullable IgniteInClosure<? super IgniteInternalFuture<?>> lsnr) throws IgniteCheckedException {
assert ses instanceof GridSelectorNioSessionImpl;
GridSelectorNioSessionImpl impl = (GridSelectorNioSessionImpl)ses;
- NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
- skipRecoveryPred.apply(msg));
-
if (lsnr != null) {
+ NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, NioOperation.REQUIRE_WRITE, msg,
+ skipRecoveryPred.apply(msg));
+
fut.listen(lsnr);
assert !fut.isDone();
- }
- send0(impl, fut, true);
+ send0(impl, fut, true);
+ }
+ else {
+ SessionWriteRequest req = new WriteRequestSystemImpl(ses, msg);
- return fut;
+ send0(impl, req, true);
+ }
}
/**
@@ -504,37 +609,69 @@ public class GridNioServer<T> {
public void resend(GridNioSession ses) {
assert ses instanceof GridSelectorNioSessionImpl;
- GridNioRecoveryDescriptor recoveryDesc = ses.recoveryDescriptor();
+ GridNioRecoveryDescriptor recoveryDesc = ses.outRecoveryDescriptor();
- if (recoveryDesc != null && !recoveryDesc.messagesFutures().isEmpty()) {
- Deque<GridNioFuture<?>> futs = recoveryDesc.messagesFutures();
+ if (recoveryDesc != null && !recoveryDesc.messagesRequests().isEmpty()) {
+ Deque<SessionWriteRequest> futs = recoveryDesc.messagesRequests();
if (log.isDebugEnabled())
log.debug("Resend messages [rmtNode=" + recoveryDesc.node().id() + ", msgCnt=" + futs.size() + ']');
GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
- GridNioFuture<?> fut0 = futs.iterator().next();
+ SessionWriteRequest fut0 = futs.iterator().next();
- for (GridNioFuture<?> fut : futs) {
+ for (SessionWriteRequest fut : futs) {
fut.messageThread(true);
- ((NioOperationFuture)fut).resetSession(ses0);
+ fut.resetSession(ses0);
}
ses0.resend(futs);
// Wake up worker.
- clientWorkers.get(ses0.selectorIndex()).offer(((NioOperationFuture)fut0));
+ ses0.offerStateChange((GridNioServer.SessionChangeRequest)fut0);
}
}
/**
+ * @return Sessions.
+ */
+ public Collection<? extends GridNioSession> sessions() {
+ return sessions;
+ }
+
+ /**
+ * @return Workers.
+ */
+ public List<AbstractNioClientWorker> workers() {
+ return clientWorkers;
+ }
+
+ /**
+ * @param ses Session.
+ * @param from Move from index.
+ * @param to Move to index.
+ */
+ private void moveSession(GridNioSession ses, int from, int to) {
+ assert from >= 0 && from < clientWorkers.size() : from;
+ assert to >= 0 && to < clientWorkers.size() : to;
+ assert from != to;
+
+ GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
+
+ SessionMoveFuture fut = new SessionMoveFuture(ses0, to);
+
+ if (!ses0.offerMove(clientWorkers.get(from), fut))
+ fut.onDone(false);
+ }
+
+ /**
* @param ses Session.
* @param op Operation.
* @return Future for operation.
*/
- GridNioFuture<?> pauseResumeReads(GridNioSession ses, NioOperation op) {
+ private GridNioFuture<?> pauseResumeReads(GridNioSession ses, NioOperation op) {
assert ses instanceof GridSelectorNioSessionImpl;
assert op == NioOperation.PAUSE_READ || op == NioOperation.RESUME_READ;
@@ -546,7 +683,7 @@ public class GridNioServer<T> {
NioOperationFuture<?> fut = new NioOperationFuture<Void>(impl, op);
- clientWorkers.get(impl.selectorIndex()).offer(fut);
+ impl.offerStateChange(fut);
return fut;
}
@@ -555,6 +692,9 @@ public class GridNioServer<T> {
*
*/
public void dumpStats() {
+ U.warn(log, "NIO server statistics [readerSesBalanceCnt=" + readerMoveCnt.get() +
+ ", writerSesBalanceCnt=" + writerMoveCnt.get() + ']');
+
for (int i = 0; i < clientWorkers.size(); i++)
clientWorkers.get(i).offer(new NioOperationFuture<Void>(null, NioOperation.DUMP_STATS));
}
@@ -675,12 +815,35 @@ public class GridNioServer<T> {
* @param req Request to balance.
*/
private synchronized void offerBalanced(NioOperationFuture req) {
- clientWorkers.get(balanceIdx).offer(req);
+ assert req.operation() == NioOperation.REGISTER : req;
+ assert req.socketChannel() != null : req;
+
+ int workers = clientWorkers.size();
+
+ int balanceIdx;
+
+ if (workers > 1) {
+ if (req.accepted()) {
+ balanceIdx = readBalanceIdx;
+
+ readBalanceIdx += 2;
+
+ if (readBalanceIdx >= workers)
+ readBalanceIdx = 0;
+ }
+ else {
+ balanceIdx = writeBalanceIdx;
- balanceIdx++;
+ writeBalanceIdx += 2;
- if (balanceIdx == clientWorkers.size())
+ if (writeBalanceIdx >= workers)
+ writeBalanceIdx = 1;
+ }
+ }
+ else
balanceIdx = 0;
+
+ clientWorkers.get(balanceIdx).offer(req);
}
/** {@inheritDoc} */
@@ -792,21 +955,30 @@ public class GridNioServer<T> {
while (true) {
ByteBuffer buf = ses.removeMeta(BUF_META_KEY);
- NioOperationFuture<?> req = ses.removeMeta(NIO_OPERATION.ordinal());
+ SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
// Check if there were any pending data from previous writes.
if (buf == null) {
assert req == null;
- req = (NioOperationFuture<?>)ses.pollFuture();
+ req = ses.pollFuture();
if (req == null) {
- key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+ if (ses.procWrite.get()) {
+ ses.procWrite.set(false);
+
+ if (ses.writeQueue().isEmpty()) {
+ if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
+ key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+ }
+ else
+ ses.procWrite.set(true);
+ }
break;
}
- buf = req.message();
+ buf = (ByteBuffer)req.message();
}
if (!skipWrite) {
@@ -841,10 +1013,15 @@ public class GridNioServer<T> {
// Message was successfully written.
assert req != null;
- req.onDone();
+ req.onMessageWritten();
}
}
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ByteBufferNioClientWorker.class, this, super.toString());
+ }
}
/**
@@ -909,6 +1086,7 @@ public class GridNioServer<T> {
metricsLsnr.onBytesReceived(cnt);
ses.bytesReceived(cnt);
+ onRead(cnt);
readBuf.flip();
@@ -921,6 +1099,12 @@ public class GridNioServer<T> {
readBuf.compact();
else
readBuf.clear();
+
+ if (ses.hasSystemMessage() && !ses.procWrite.get()) {
+ ses.procWrite.set(true);
+
+ registerWrite(ses);
+ }
}
catch (IgniteCheckedException e) {
close(ses, e);
@@ -993,16 +1177,29 @@ public class GridNioServer<T> {
if (ses.meta(WRITE_BUF_LIMIT) != null)
buf.limit((int)ses.meta(WRITE_BUF_LIMIT));
- NioOperationFuture<?> req = ses.removeMeta(NIO_OPERATION.ordinal());
+ SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
while (true) {
if (req == null) {
- req = (NioOperationFuture<?>)ses.pollFuture();
+ req = systemMessage(ses);
- if (req == null && buf.position() == 0) {
- key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+ if (req == null) {
+ req = ses.pollFuture();
- break;
+ if (req == null && buf.position() == 0) {
+ if (ses.procWrite.get()) {
+ ses.procWrite.set(false);
+
+ if (ses.writeQueue().isEmpty()) {
+ if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
+ key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+ }
+ else
+ ses.procWrite.set(true);
+ }
+
+ break;
+ }
}
}
@@ -1010,7 +1207,7 @@ public class GridNioServer<T> {
boolean finished = false;
if (req != null) {
- msg = req.directMessage();
+ msg = (Message)req.message();
assert msg != null;
@@ -1025,14 +1222,17 @@ public class GridNioServer<T> {
// Fill up as many messages as possible to write buffer.
while (finished) {
- req.onDone();
+ req.onMessageWritten();
- req = (NioOperationFuture<?>)ses.pollFuture();
+ req = systemMessage(ses);
+
+ if (req == null)
+ req = ses.pollFuture();
if (req == null)
break;
- msg = req.directMessage();
+ msg = (Message)req.message();
assert msg != null;
@@ -1129,13 +1329,31 @@ public class GridNioServer<T> {
ses.bytesSent(cnt);
if (!buf.hasRemaining())
- queue.remove(buf);
+ queue.poll();
else
break;
}
}
/**
+ * @param ses Session.
+ * @return System message request.
+ */
+ private SessionWriteRequest systemMessage(GridSelectorNioSessionImpl ses) {
+ if (ses.hasSystemMessage()) {
+ Object msg = ses.systemMessage();
+
+ SessionWriteRequest req = new WriteRequestSystemImpl(ses, msg);
+
+ assert !ses.hasSystemMessage();
+
+ return req;
+ }
+
+ return null;
+ }
+
+ /**
* Processes write-ready event on the key.
*
* @param key Key that is ready to be written.
@@ -1147,7 +1365,7 @@ public class GridNioServer<T> {
GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
ByteBuffer buf = ses.writeBuffer();
- NioOperationFuture<?> req = ses.removeMeta(NIO_OPERATION.ordinal());
+ SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
MessageWriter writer = ses.meta(MSG_WRITER.ordinal());
@@ -1161,12 +1379,25 @@ public class GridNioServer<T> {
}
if (req == null) {
- req = (NioOperationFuture<?>)ses.pollFuture();
+ req = systemMessage(ses);
- if (req == null && buf.position() == 0) {
- key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+ if (req == null) {
+ req = ses.pollFuture();
- return;
+ if (req == null && buf.position() == 0) {
+ if (ses.procWrite.get()) {
+ ses.procWrite.set(false);
+
+ if (ses.writeQueue().isEmpty()) {
+ if ((key.interestOps() & SelectionKey.OP_WRITE) != 0)
+ key.interestOps(key.interestOps() & (~SelectionKey.OP_WRITE));
+ }
+ else
+ ses.procWrite.set(true);
+ }
+
+ return;
+ }
}
}
@@ -1174,9 +1405,9 @@ public class GridNioServer<T> {
boolean finished = false;
if (req != null) {
- msg = req.directMessage();
+ msg = (Message)req.message();
- assert msg != null;
+ assert msg != null : req;
if (writer != null)
writer.setCurrentWriteClass(msg.getClass());
@@ -1189,14 +1420,17 @@ public class GridNioServer<T> {
// Fill up as many messages as possible to write buffer.
while (finished) {
- req.onDone();
+ req.onMessageWritten();
- req = (NioOperationFuture<?>)ses.pollFuture();
+ req = systemMessage(ses);
+
+ if (req == null)
+ req = ses.pollFuture();
if (req == null)
break;
- msg = req.directMessage();
+ msg = (Message)req.message();
assert msg != null;
@@ -1223,6 +1457,7 @@ public class GridNioServer<T> {
metricsLsnr.onBytesSent(cnt);
ses.bytesSent(cnt);
+ onWrite(cnt);
}
else {
// For test purposes only (skipWrite is set to true in tests only).
@@ -1242,14 +1477,19 @@ public class GridNioServer<T> {
else
buf.clear();
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DirectNioClientWorker.class, this, super.toString());
+ }
}
/**
* Thread performing only read operations from the channel.
*/
- private abstract class AbstractNioClientWorker extends GridWorker {
+ private abstract class AbstractNioClientWorker extends GridWorker implements GridNioWorker {
/** Queue of change requests on this selector. */
- private final ConcurrentLinkedQueue<NioOperationFuture> changeReqs = new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue<SessionChangeRequest> changeReqs = new ConcurrentLinkedQueue<>();
/** Selector to select read events. */
private Selector selector;
@@ -1260,6 +1500,25 @@ public class GridNioServer<T> {
/** Worker index. */
private final int idx;
+ /** */
+ private long bytesRcvd;
+
+ /** */
+ private long bytesSent;
+
+ /** */
+ private volatile long bytesRcvd0;
+
+ /** */
+ private volatile long bytesSent0;
+
+ /** Sessions assigned to this worker. */
+ private final GridConcurrentHashSet<GridSelectorNioSessionImpl> workerSessions =
+ new GridConcurrentHashSet<>();
+
+ /** {@code True} if worker has called or is about to call {@code Selector.select()}. */
+ private volatile boolean select;
+
/**
* @param idx Index of this worker in server's array.
* @param gridName Grid name.
@@ -1322,15 +1581,15 @@ public class GridNioServer<T> {
try {
SelectedSelectionKeySet selectedKeySet = new SelectedSelectionKeySet();
- Class<?> selectorImplClass =
+ Class<?> selectorImplCls =
Class.forName("sun.nio.ch.SelectorImpl", false, U.gridClassLoader());
// Ensure the current selector implementation is what we can instrument.
- if (!selectorImplClass.isAssignableFrom(selector.getClass()))
+ if (!selectorImplCls.isAssignableFrom(selector.getClass()))
return;
- Field selectedKeysField = selectorImplClass.getDeclaredField("selectedKeys");
- Field publicSelectedKeysField = selectorImplClass.getDeclaredField("publicSelectedKeys");
+ Field selectedKeysField = selectorImplCls.getDeclaredField("selectedKeys");
+ Field publicSelectedKeysField = selectorImplCls.getDeclaredField("publicSelectedKeys");
selectedKeysField.setAccessible(true);
publicSelectedKeysField.setAccessible(true);
@@ -1357,48 +1616,126 @@ public class GridNioServer<T> {
*
* @param req Change request.
*/
- private void offer(NioOperationFuture req) {
+ public void offer(SessionChangeRequest req) {
changeReqs.offer(req);
+ if (select)
+ selector.wakeup();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void offer(Collection<SessionChangeRequest> reqs) {
+ for (SessionChangeRequest req : reqs)
+ changeReqs.offer(req);
+
selector.wakeup();
}
+ /** {@inheritDoc} */
+ @Override public List<SessionChangeRequest> clearSessionRequests(GridNioSession ses) {
+ List<SessionChangeRequest> sesReqs = null;
+
+ for (SessionChangeRequest changeReq : changeReqs) {
+ if (changeReq.session() == ses && !(changeReq instanceof SessionMoveFuture)) {
+ boolean rmv = changeReqs.remove(changeReq);
+
+ assert rmv : changeReq;
+
+ if (sesReqs == null)
+ sesReqs = new ArrayList<>();
+
+ sesReqs.add(changeReq);
+ }
+ }
+
+ return sesReqs;
+ }
+
/**
* Processes read and write events and registration requests.
*
* @throws IgniteCheckedException If IOException occurred or thread was unable to add worker to workers pool.
*/
@SuppressWarnings("unchecked")
- private void bodyInternal() throws IgniteCheckedException {
+ private void bodyInternal() throws IgniteCheckedException, InterruptedException {
try {
long lastIdleCheck = U.currentTimeMillis();
+ mainLoop:
while (!closed && selector.isOpen()) {
- NioOperationFuture req;
+ SessionChangeRequest req0;
- while ((req = changeReqs.poll()) != null) {
- switch (req.operation()) {
+ while ((req0 = changeReqs.poll()) != null) {
+ switch (req0.operation()) {
case REGISTER: {
- register(req);
+ register((NioOperationFuture)req0);
break;
}
- case REQUIRE_WRITE: {
- //Just register write key.
- SelectionKey key = req.session().key();
+ case MOVE: {
+ SessionMoveFuture f = (SessionMoveFuture)req0;
- if (key.isValid()) {
- key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+ GridSelectorNioSessionImpl ses = f.session();
+
+ if (idx == f.toIdx) {
+ assert f.movedSocketChannel() != null : f;
+
+ boolean add = workerSessions.add(ses);
+
+ assert add;
- // Update timestamp to protected against false write timeout.
- ((GridNioSessionImpl)key.attachment()).bytesSent(0);
+ ses.finishMoveSession(this);
+
+ if (idx % 2 == 0)
+ readerMoveCnt.incrementAndGet();
+ else
+ writerMoveCnt.incrementAndGet();
+
+ SelectionKey key = f.movedSocketChannel().register(selector,
+ SelectionKey.OP_READ | SelectionKey.OP_WRITE,
+ ses);
+
+ ses.key(key);
+
+ ses.procWrite.set(true);
+
+ f.onDone(true);
+ }
+ else {
+ assert f.movedSocketChannel() == null : f;
+
+ if (workerSessions.remove(ses)) {
+ ses.startMoveSession(this);
+
+ SelectionKey key = ses.key();
+
+ assert key.channel() != null : key;
+
+ f.movedSocketChannel((SocketChannel)key.channel());
+
+ key.cancel();
+
+ clientWorkers.get(f.toIndex()).offer(f);
+ }
+ else
+ f.onDone(false);
}
break;
}
+ case REQUIRE_WRITE: {
+ SessionWriteRequest req = (SessionWriteRequest)req0;
+
+ registerWrite((GridSelectorNioSessionImpl)req.session());
+
+ break;
+ }
+
case CLOSE: {
+ NioOperationFuture req = (NioOperationFuture)req0;
+
if (close(req.session(), null))
req.onDone(true);
else
@@ -1408,6 +1745,8 @@ public class GridNioServer<T> {
}
case PAUSE_READ: {
+ NioOperationFuture req = (NioOperationFuture)req0;
+
SelectionKey key = req.session().key();
if (key.isValid()) {
@@ -1426,6 +1765,8 @@ public class GridNioServer<T> {
}
case RESUME_READ: {
+ NioOperationFuture req = (NioOperationFuture)req0;
+
SelectionKey key = req.session().key();
if (key.isValid()) {
@@ -1444,76 +1785,66 @@ public class GridNioServer<T> {
}
case DUMP_STATS: {
- StringBuilder sb = new StringBuilder();
+ NioOperationFuture req = (NioOperationFuture)req0;
- Set<SelectionKey> keys = selector.keys();
-
- sb.append(U.nl())
- .append(">> Selector info [idx=").append(idx)
- .append(", keysCnt=").append(keys.size())
- .append("]").append(U.nl());
-
- for (SelectionKey key : keys) {
- GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
-
- MessageWriter writer = ses.meta(MSG_WRITER.ordinal());
- MessageReader reader = ses.meta(GridDirectParser.READER_META_KEY);
-
- sb.append(" Connection info [")
- .append("rmtAddr=").append(ses.remoteAddress())
- .append(", locAddr=").append(ses.localAddress());
-
- GridNioRecoveryDescriptor desc = ses.recoveryDescriptor();
+ try {
+ dumpStats();
+ }
+ finally {
+ // Complete the request just in case (none should wait on this future).
+ req.onDone(true);
+ }
+ }
+ }
+ }
- if (desc != null) {
- sb.append(", msgsSent=").append(desc.sent())
- .append(", msgsAckedByRmt=").append(desc.acked())
- .append(", msgsRcvd=").append(desc.received())
- .append(", descIdHash=").append(System.identityHashCode(desc));
- }
- else
- sb.append(", recoveryDesc=null");
+ int res = 0;
- sb.append(", bytesRcvd=").append(ses.bytesReceived())
- .append(", bytesSent=").append(ses.bytesSent())
- .append(", opQueueSize=").append(ses.writeQueueSize())
- .append(", msgWriter=").append(writer != null ? writer.toString() : "null")
- .append(", msgReader=").append(reader != null ? reader.toString() : "null");
+ for (long i = 0; i < selectorSpins && res == 0; i++) {
+ res = selector.selectNow();
- int cnt = 0;
+ if (res > 0) {
+ // Walk through the ready keys collection and process network events.
+ if (selectedKeys == null)
+ processSelectedKeys(selector.selectedKeys());
+ else
+ processSelectedKeysOptimized(selectedKeys.flip());
+ }
- for (GridNioFuture<?> fut : ses.writeQueue()) {
- if (cnt == 0)
- sb.append(",\n opQueue=[").append(fut);
- else
- sb.append(',').append(fut);
+ if (!changeReqs.isEmpty())
+ continue mainLoop;
- if (++cnt == 5) {
- sb.append(']');
+ // Just in case we do busy selects.
+ long now = U.currentTimeMillis();
- break;
- }
- }
+ if (now - lastIdleCheck > 2000) {
+ lastIdleCheck = now;
+ checkIdle(selector.keys());
+ }
- sb.append("]").append(U.nl());
- }
+ if (isCancelled())
+ return;
+ }
- U.warn(log, sb.toString());
+ // Falling to blocking select.
+ select = true;
- // Complete the request just in case (none should wait on this future).
- req.onDone(true);
- }
+ try {
+ if (!changeReqs.isEmpty())
+ continue;
+
+ // Wake up every 2 seconds to check if closed.
+ if (selector.select(2000) > 0) {
+ // Walk through the ready keys collection and process network events.
+ if (selectedKeys == null)
+ processSelectedKeys(selector.selectedKeys());
+ else
+ processSelectedKeysOptimized(selectedKeys.flip());
}
}
-
- // Wake up every 2 seconds to check if closed.
- if (selector.select(2000) > 0) {
- // Walk through the ready keys collection and process network events.
- if (selectedKeys == null)
- processSelectedKeys(selector.selectedKeys());
- else
- processSelectedKeysOptimized(selectedKeys.flip());
+ finally {
+ select = false;
}
long now = U.currentTimeMillis();
@@ -1554,6 +1885,98 @@ public class GridNioServer<T> {
}
/**
+ * @param ses Session.
+ */
+ public final void registerWrite(GridSelectorNioSessionImpl ses) {
+ SelectionKey key = ses.key();
+
+ if (key.isValid()) {
+ if ((key.interestOps() & SelectionKey.OP_WRITE) == 0)
+ key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+
+ // Update timestamp to protected against false write timeout.
+ ses.bytesSent(0);
+ }
+ }
+
+ /**
+ *
+ */
+ private void dumpStats() {
+ StringBuilder sb = new StringBuilder();
+
+ Set<SelectionKey> keys = selector.keys();
+
+ sb.append(U.nl())
+ .append(">> Selector info [idx=").append(idx)
+ .append(", keysCnt=").append(keys.size())
+ .append(", bytesRcvd=").append(bytesRcvd)
+ .append(", bytesRcvd0=").append(bytesRcvd0)
+ .append(", bytesSent=").append(bytesSent)
+ .append(", bytesSent0=").append(bytesSent0)
+ .append("]").append(U.nl());
+
+ for (SelectionKey key : keys) {
+ GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+
+ MessageWriter writer = ses.meta(MSG_WRITER.ordinal());
+ MessageReader reader = ses.meta(GridDirectParser.READER_META_KEY);
+
+ sb.append(" Connection info [")
+ .append("in=").append(ses.accepted())
+ .append(", rmtAddr=").append(ses.remoteAddress())
+ .append(", locAddr=").append(ses.localAddress());
+
+ GridNioRecoveryDescriptor outDesc = ses.outRecoveryDescriptor();
+
+ if (outDesc != null) {
+ sb.append(", msgsSent=").append(outDesc.sent())
+ .append(", msgsAckedByRmt=").append(outDesc.acked())
+ .append(", descIdHash=").append(System.identityHashCode(outDesc));
+ }
+ else
+ sb.append(", outRecoveryDesc=null");
+
+ GridNioRecoveryDescriptor inDesc = ses.inRecoveryDescriptor();
+
+ if (inDesc != null) {
+ sb.append(", msgsRcvd=").append(inDesc.received())
+ .append(", lastAcked=").append(inDesc.lastAcknowledged())
+ .append(", descIdHash=").append(System.identityHashCode(inDesc));
+ }
+ else
+ sb.append(", inRecoveryDesc=null");
+
+ sb.append(", bytesRcvd=").append(ses.bytesReceived())
+ .append(", bytesRcvd0=").append(ses.bytesReceived0())
+ .append(", bytesSent=").append(ses.bytesSent())
+ .append(", bytesSent0=").append(ses.bytesSent0())
+ .append(", opQueueSize=").append(ses.writeQueueSize())
+ .append(", msgWriter=").append(writer != null ? writer.toString() : "null")
+ .append(", msgReader=").append(reader != null ? reader.toString() : "null");
+
+ int cnt = 0;
+
+ for (SessionWriteRequest req : ses.writeQueue()) {
+ if (cnt == 0)
+ sb.append(",\n opQueue=[").append(req);
+ else
+ sb.append(',').append(req);
+
+ if (++cnt == 5) {
+ sb.append(']');
+
+ break;
+ }
+ }
+
+ sb.append("]").append(U.nl());
+ }
+
+ U.warn(log, sb.toString());
+ }
+
+ /**
* Processes keys selected by a selector.
*
* @param keys Selected keys.
@@ -1671,7 +2094,9 @@ public class GridNioServer<T> {
long idleTimeout0 = idleTimeout;
- if (!opWrite && now - ses.lastReceiveTime() > idleTimeout0 && now - ses.lastSendScheduleTime() > idleTimeout0) {
+ if (!opWrite &&
+ now - ses.lastReceiveTime() > idleTimeout0 &&
+ now - ses.lastSendScheduleTime() > idleTimeout0) {
filterChain.onSessionIdleTimeout(ses);
// Update timestamp to avoid multiple notifications within one timeout interval.
@@ -1715,7 +2140,7 @@ public class GridNioServer<T> {
final GridSelectorNioSessionImpl ses = new GridSelectorNioSessionImpl(
log,
- idx,
+ this,
filterChain,
(InetSocketAddress)sockCh.getLocalAddress(),
(InetSocketAddress)sockCh.getRemoteAddress(),
@@ -1739,6 +2164,7 @@ public class GridNioServer<T> {
resend(ses);
sessions.add(ses);
+ workerSessions.add(ses);
try {
filterChain.onSessionOpened(ses);
@@ -1764,7 +2190,7 @@ public class GridNioServer<T> {
}
/**
- * Closes the ses and all associated resources, then notifies the listener.
+ * Closes the session and all associated resources, then notifies the listener.
*
* @param ses Session to be closed.
* @param e Exception to be passed to the listener, if any.
@@ -1781,12 +2207,10 @@ public class GridNioServer<T> {
}
sessions.remove(ses);
+ workerSessions.remove(ses);
SelectionKey key = ses.key();
- // Shutdown input and output so that remote client will see correct socket close.
- Socket sock = ((SocketChannel)key.channel()).socket();
-
if (ses.setClosed()) {
ses.onClosed();
@@ -1798,6 +2222,9 @@ public class GridNioServer<T> {
((DirectBuffer)ses.readBuffer()).cleaner().clean();
}
+ // Shutdown input and output so that remote client will see correct socket close.
+ Socket sock = ((SocketChannel)key.channel()).socket();
+
try {
try {
sock.shutdownInput();
@@ -1824,28 +2251,35 @@ public class GridNioServer<T> {
ses.removeMeta(BUF_META_KEY);
// Since ses is in closed state, no write requests will be added.
- NioOperationFuture<?> fut = ses.removeMeta(NIO_OPERATION.ordinal());
+ SessionWriteRequest req = ses.removeMeta(NIO_OPERATION.ordinal());
+
+ GridNioRecoveryDescriptor outRecovery = ses.outRecoveryDescriptor();
+ GridNioRecoveryDescriptor inRecovery = ses.inRecoveryDescriptor();
- GridNioRecoveryDescriptor recovery = ses.recoveryDescriptor();
+ IOException err = new IOException("Failed to send message (connection was closed): " + ses);
- if (recovery != null) {
+ if (outRecovery != null || inRecovery != null) {
try {
// Poll will update recovery data.
- while ((fut = (NioOperationFuture<?>)ses.pollFuture()) != null) {
- if (fut.skipRecovery())
- fut.connectionClosed();
+ while ((req = ses.pollFuture()) != null) {
+ if (req.skipRecovery())
+ req.onError(err);
}
}
finally {
- recovery.release();
+ if (outRecovery != null)
+ outRecovery.release();
+
+ if (inRecovery != null && inRecovery != outRecovery)
+ inRecovery.release();
}
}
else {
- if (fut != null)
- fut.connectionClosed();
+ if (req != null)
+ req.onError(err);
- while ((fut = (NioOperationFuture<?>)ses.pollFuture()) != null)
- fut.connectionClosed();
+ while ((req = ses.pollFuture()) != null)
+ req.onError(err);
}
try {
@@ -1876,12 +2310,44 @@ public class GridNioServer<T> {
* @throws IOException If write failed.
*/
protected abstract void processWrite(SelectionKey key) throws IOException;
- }
- /**
- * Gets outbound messages queue size.
- *
- * @return Write queue size.
+ /**
+ * @param cnt
+ */
+ final void onRead(int cnt) {
+ bytesRcvd += cnt;
+ bytesRcvd0 += cnt;
+ }
+
+ /**
+ * @param cnt
+ */
+ final void onWrite(int cnt) {
+ bytesSent += cnt;
+ bytesSent0 += cnt;
+ }
+
+ /**
+ *
+ */
+ final void reset0() {
+ bytesSent0 = 0;
+ bytesRcvd0 = 0;
+
+ for (GridSelectorNioSessionImpl ses : workerSessions)
+ ses.reset0();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(AbstractNioClientWorker.class, this, super.toString());
+ }
+ }
+
+ /**
+ * Gets outbound messages queue size.
+ *
+ * @return Write queue size.
*/
public int outboundMessagesQueueSize() {
int res = 0;
@@ -1952,6 +2418,9 @@ public class GridNioServer<T> {
if (selector.select(2000) > 0)
// Walk through the ready keys collection and process date requests.
processSelectedKeys(selector.selectedKeys());
+
+ if (balancer != null)
+ balancer.run();
}
}
// Ignore this exception as thread interruption is equal to 'close' call.
@@ -2048,10 +2517,13 @@ public class GridNioServer<T> {
/**
* Asynchronous operation that may be requested on selector.
*/
- private enum NioOperation {
+ enum NioOperation {
/** Register read key selection. */
REGISTER,
+ /** Move session between workers. */
+ MOVE,
+
/** Register write key selection. */
REQUIRE_WRITE,
@@ -2069,9 +2541,193 @@ public class GridNioServer<T> {
}
/**
+ *
+ */
+ private static final class WriteRequestSystemImpl implements SessionWriteRequest, SessionChangeRequest {
+ /** */
+ private final Object msg;
+
+ /** */
+ private final GridNioSession ses;
+
+ /**
+ * @param ses Session.
+ * @param msg Message.
+ */
+ WriteRequestSystemImpl(GridNioSession ses, Object msg) {
+ this.ses = ses;
+ this.msg = msg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void messageThread(boolean msgThread) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean messageThread() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean skipRecovery() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void ackClosure(IgniteInClosure<IgniteException> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInClosure<IgniteException> ackClosure() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onError(Exception e) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object message() {
+ return msg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessageWritten() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resetSession(GridNioSession ses) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioSession session() {
+ return ses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public NioOperation operation() {
+ return NioOperation.REQUIRE_WRITE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(WriteRequestSystemImpl.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private static final class WriteRequestImpl implements SessionWriteRequest, SessionChangeRequest {
+ /** */
+ private GridNioSession ses;
+
+ /** */
+ private final Object msg;
+
+ /** */
+ private boolean msgThread;
+
+ /** */
+ private final boolean skipRecovery;
+
+ /** */
+ private IgniteInClosure<IgniteException> ackC;
+
+ /**
+ * @param ses Session.
+ * @param msg Message.
+ * @param skipRecovery Skip recovery flag.
+ */
+ WriteRequestImpl(GridNioSession ses, Object msg, boolean skipRecovery) {
+ this.ses = ses;
+ this.msg = msg;
+ this.skipRecovery = skipRecovery;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void messageThread(boolean msgThread) {
+ this.msgThread = msgThread;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean messageThread() {
+ return msgThread;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean skipRecovery() {
+ return skipRecovery;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void ackClosure(IgniteInClosure<IgniteException> c) {
+ ackC = c;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ assert msg instanceof Message;
+
+ ((Message)msg).onAckReceived();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInClosure<IgniteException> ackClosure() {
+ return ackC;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onError(Exception e) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object message() {
+ return msg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessageWritten() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resetSession(GridNioSession ses) {
+ this.ses = ses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridNioSession session() {
+ return ses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public NioOperation operation() {
+ return NioOperation.REQUIRE_WRITE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(WriteRequestImpl.class, this);
+ }
+ }
+
+ /**
* Class for requesting write and session close operations.
*/
- private static class NioOperationFuture<R> extends GridNioFutureImpl<R> {
+ private static class NioOperationFuture<R> extends GridNioFutureImpl<R> implements SessionWriteRequest,
+ SessionChangeRequest {
/** */
private static final long serialVersionUID = 0L;
@@ -2087,11 +2743,7 @@ public class GridNioServer<T> {
private NioOperation op;
/** Message. */
- @GridToStringExclude
- private ByteBuffer msg;
-
- /** Direct message. */
- private Message commMsg;
+ private Object msg;
/** */
@GridToStringExclude
@@ -2153,8 +2805,7 @@ public class GridNioServer<T> {
* @param op Requested operation.
* @param msg Message.
*/
- NioOperationFuture(GridSelectorNioSessionImpl ses, NioOperation op,
- ByteBuffer msg) {
+ NioOperationFuture(GridSelectorNioSessionImpl ses, NioOperation op, Object msg) {
assert ses != null;
assert op != null;
assert op != NioOperation.REGISTER;
@@ -2182,51 +2833,36 @@ public class GridNioServer<T> {
this.ses = ses;
this.op = op;
- this.commMsg = commMsg;
+ this.msg = commMsg;
this.skipRecovery = skipRecovery;
}
- /**
- * @return Requested change operation.
- */
- private NioOperation operation() {
+ /** {@inheritDoc} */
+ public NioOperation operation() {
return op;
}
- /**
- * @return Message.
- */
- private ByteBuffer message() {
+ /** {@inheritDoc} */
+ public Object message() {
return msg;
}
- /**
- * @return Direct message.
- */
- private Message directMessage() {
- return commMsg;
- }
-
- /**
- * @param ses New session instance.
- */
- private void resetSession(GridSelectorNioSessionImpl ses) {
- assert commMsg != null;
+ /** {@inheritDoc} */
+ public void resetSession(GridNioSession ses) {
+ assert msg instanceof Message : msg;
- this.ses = ses;
+ this.ses = (GridSelectorNioSessionImpl)ses;
}
/**
* @return Socket channel for register request.
*/
- private SocketChannel socketChannel() {
+ SocketChannel socketChannel() {
return sockCh;
}
- /**
- * @return Session for this change request.
- */
- private GridSelectorNioSessionImpl session() {
+ /** {@inheritDoc} */
+ public GridSelectorNioSessionImpl session() {
return ses;
}
@@ -2244,21 +2880,21 @@ public class GridNioServer<T> {
return meta;
}
- /**
- * Applicable to write futures only. Fails future with corresponding IOException.
- */
- private void connectionClosed() {
- assert op == NioOperation.REQUIRE_WRITE;
- assert ses != null;
-
- onDone(new IOException("Failed to send message (connection was closed): " + ses));
+ /** {@inheritDoc} */
+ @Override public void onError(Exception e) {
+ onDone(e);
}
/** {@inheritDoc} */
@Override public void onAckReceived() {
- assert commMsg != null;
+ assert msg instanceof Message : msg;
- commMsg.onAckReceived();
+ ((Message)msg).onAckReceived();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onMessageWritten() {
+ onDone();
}
/** {@inheritDoc} */
@@ -2273,6 +2909,59 @@ public class GridNioServer<T> {
}
/**
+ *
+ */
+ private static class SessionMoveFuture extends NioOperationFuture<Boolean> {
+ /** */
+ private final int toIdx;
+
+ /** */
+ @GridToStringExclude
+ private SocketChannel movedSockCh;
+
+ /**
+ * @param ses Session.
+ * @param toIdx Target worker index.
+ */
+ SessionMoveFuture(
+ GridSelectorNioSessionImpl ses,
+ int toIdx
+ ) {
+ super(ses, NioOperation.MOVE);
+
+ this.toIdx = toIdx;
+ }
+
+ /**
+ * @return Target worker index.
+ */
+ int toIndex() {
+ return toIdx;
+ }
+
+ /**
+ * @return Moved session socket channel.
+ */
+ SocketChannel movedSocketChannel() {
+ return movedSockCh;
+ }
+
+ /**
+ * @param movedSockCh Moved session socket channel.
+ */
+ void movedSocketChannel(SocketChannel movedSockCh) {
+ assert movedSockCh != null;
+
+ this.movedSockCh = movedSockCh;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(SessionMoveFuture.class, this, super.toString());
+ }
+ }
+
+ /**
* Filter forwarding messages from chain's head to this server.
*/
private class HeadFilter extends GridNioFilterAdapter {
@@ -2302,7 +2991,7 @@ public class GridNioServer<T> {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException {
if (directMode) {
boolean sslSys = sslFilter != null && msg instanceof ByteBuffer;
@@ -2313,18 +3002,18 @@ public class GridNioServer<T> {
queue.offer((ByteBuffer)msg);
- SelectionKey key = ((GridSelectorNioSessionImpl)ses).key();
+ GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
- if (key.isValid())
- key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+ if (!ses0.procWrite.get() && ses0.procWrite.compareAndSet(false, true))
+ ses0.worker().registerWrite(ses0);
return null;
}
else
- return send(ses, (Message)msg);
+ return send(ses, (Message)msg, fut);
}
else
- return send(ses, (ByteBuffer)msg);
+ return send(ses, (ByteBuffer)msg, fut);
}
/** {@inheritDoc} */
@@ -2429,6 +3118,15 @@ public class GridNioServer<T> {
/** Message queue size listener. */
private IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr;
+ /** Name for threads identification. */
+ private String srvName;
+
+ /** */
+ private long selectorSpins;
+
+ /** NIO sessions balancing flag. */
+ private boolean balancing;
+
/**
* Finishes building the instance.
*
@@ -2442,6 +3140,8 @@ public class GridNioServer<T> {
log,
selectorCnt,
gridName,
+ srvName,
+ selectorSpins,
tcpNoDelay,
directBuf,
byteOrder,
@@ -2455,6 +3155,7 @@ public class GridNioServer<T> {
writerFactory,
skipRecoveryPred,
msgQueueLsnr,
+ balancing,
filters != null ? Arrays.copyOf(filters, filters.length) : EMPTY_FILTERS
);
@@ -2468,6 +3169,16 @@ public class GridNioServer<T> {
}
/**
+ * @param balancing NIO sessions balancing flag.
+ * @return This for chaining.
+ */
+ public Builder<T> balancing(boolean balancing) {
+ this.balancing = balancing;
+
+ return this;
+ }
+
+ /**
* @param addr Local address.
* @return This for chaining.
*/
@@ -2519,6 +3230,28 @@ public class GridNioServer<T> {
}
/**
+ * @param srvName Logical server name for threads identification.
+ * @return This for chaining.
+ */
+ public Builder<T> serverName(@Nullable String srvName) {
+ this.srvName = srvName;
+
+ return this;
+ }
+
+ /**
+ * @param selectorSpins Defines how many non-blocking {@code selector.selectNow()} should be made before
+ * falling into {@code selector.select(long)} in NIO server. Long value. Default is {@code 0}.
+ * Can be set to {@code Long.MAX_VALUE} so selector threads will never block.
+ * @return This for chaining.
+ */
+ public Builder<T> selectorSpins(long selectorSpins) {
+ this.selectorSpins = selectorSpins;
+
+ return this;
+ }
+
+ /**
* @param tcpNoDelay If TCP_NODELAY option should be set to accepted sockets.
* @return This for chaining.
*/
@@ -2678,4 +3411,225 @@ public class GridNioServer<T> {
return this;
}
}
+
+ /**
+ *
+ */
+ private class SizeBasedBalancer implements IgniteRunnable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private long lastBalance;
+
+ /** */
+ private final long balancePeriod;
+
+ /**
+ * @param balancePeriod Period.
+ */
+ SizeBasedBalancer(long balancePeriod) {
+ this.balancePeriod = balancePeriod;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ long now = U.currentTimeMillis();
+
+ if (lastBalance + balancePeriod < now) {
+ lastBalance = now;
+
+ long maxRcvd0 = -1, minRcvd0 = -1, maxSent0 = -1, minSent0 = -1;
+ int maxRcvdIdx = -1, minRcvdIdx = -1, maxSentIdx = -1, minSentIdx = -1;
+
+ for (int i = 0; i < clientWorkers.size(); i++) {
+ GridNioServer.AbstractNioClientWorker worker = clientWorkers.get(i);
+
+ int sesCnt = worker.workerSessions.size();
+
+ if (i % 2 == 0) {
+ // Reader.
+ long bytesRcvd0 = worker.bytesRcvd0;
+
+ if ((maxRcvd0 == -1 || bytesRcvd0 > maxRcvd0) && bytesRcvd0 > 0 && sesCnt > 1) {
+ maxRcvd0 = bytesRcvd0;
+ maxRcvdIdx = i;
+ }
+
+ if (minRcvd0 == -1 || bytesRcvd0 < minRcvd0) {
+ minRcvd0 = bytesRcvd0;
+ minRcvdIdx = i;
+ }
+ }
+ else {
+ // Writer.
+ long bytesSent0 = worker.bytesSent0;
+
+ if ((maxSent0 == -1 || bytesSent0 > maxSent0) && bytesSent0 > 0 && sesCnt > 1) {
+ maxSent0 = bytesSent0;
+ maxSentIdx = i;
+ }
+
+ if (minSent0 == -1 || bytesSent0 < minSent0) {
+ minSent0 = bytesSent0;
+ minSentIdx = i;
+ }
+ }
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Balancing data [minSent0=" + minSent0 + ", minSentIdx=" + minSentIdx +
+ ", maxSent0=" + maxSent0 + ", maxSentIdx=" + maxSentIdx +
+ ", minRcvd0=" + minRcvd0 + ", minRcvdIdx=" + minRcvdIdx +
+ ", maxRcvd0=" + maxRcvd0 + ", maxRcvdIdx=" + maxRcvdIdx + ']');
+
+ if (maxSent0 != -1 && minSent0 != -1) {
+ GridSelectorNioSessionImpl ses = null;
+
+ long sentDiff = maxSent0 - minSent0;
+ long delta = sentDiff;
+ double threshold = sentDiff * 0.9;
+
+ GridConcurrentHashSet<GridSelectorNioSessionImpl> sessions =
+ clientWorkers.get(maxSentIdx).workerSessions;
+
+ for (GridSelectorNioSessionImpl ses0 : sessions) {
+ long bytesSent0 = ses0.bytesSent0();
+
+ if (bytesSent0 < threshold &&
+ (ses == null || delta > U.safeAbs(bytesSent0 - sentDiff / 2))) {
+ ses = ses0;
+ delta = U.safeAbs(bytesSent0 - sentDiff / 2);
+ }
+ }
+
+ if (ses != null) {
+ if (log.isDebugEnabled())
+ log.debug("Will move session to less loaded writer [ses=" + ses +
+ ", from=" + maxSentIdx + ", to=" + minSentIdx + ']');
+
+ moveSession(ses, maxSentIdx, minSentIdx);
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Unable to find session to move for writers.");
+ }
+ }
+
+ if (maxRcvd0 != -1 && minRcvd0 != -1) {
+ GridSelectorNioSessionImpl ses = null;
+
+ long rcvdDiff = maxRcvd0 - minRcvd0;
+ long delta = rcvdDiff;
+ double threshold = rcvdDiff * 0.9;
+
+ GridConcurrentHashSet<GridSelectorNioSessionImpl> sessions =
+ clientWorkers.get(maxRcvdIdx).workerSessions;
+
+ for (GridSelectorNioSessionImpl ses0 : sessions) {
+ long bytesRcvd0 = ses0.bytesReceived0();
+
+ if (bytesRcvd0 < threshold &&
+ (ses == null || delta > U.safeAbs(bytesRcvd0 - rcvdDiff / 2))) {
+ ses = ses0;
+ delta = U.safeAbs(bytesRcvd0 - rcvdDiff / 2);
+ }
+ }
+
+ if (ses != null) {
+ if (log.isDebugEnabled())
+ log.debug("Will move session to less loaded reader [ses=" + ses +
+ ", from=" + maxRcvdIdx + ", to=" + minRcvdIdx + ']');
+
+ moveSession(ses, maxRcvdIdx, minRcvdIdx);
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Unable to find session to move for readers.");
+ }
+ }
+
+ for (int i = 0; i < clientWorkers.size(); i++) {
+ GridNioServer.AbstractNioClientWorker worker = clientWorkers.get(i);
+
+ worker.reset0();
+ }
+ }
+ }
+ }
+
+ /**
+ * For tests only.
+ */
+ @SuppressWarnings("unchecked")
+ private class RandomBalancer implements IgniteRunnable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int w1 = rnd.nextInt(clientWorkers.size());
+
+ if (clientWorkers.get(w1).workerSessions.isEmpty())
+ return;
+
+ int w2 = rnd.nextInt(clientWorkers.size());
+
+ while (w2 == w1)
+ w2 = rnd.nextInt(clientWorkers.size());
+
+ GridNioSession ses = randomSession(clientWorkers.get(w1));
+
+ if (ses != null) {
+ log.info("Move session [from=" + w1 +
+ ", to=" + w2 +
+ ", ses=" + ses + ']');
+
+ moveSession(ses, w1, w2);
+ }
+ }
+
+ /**
+ * @param worker Worker.
+ * @return NIO session.
+ */
+ private GridNioSession randomSession(GridNioServer.AbstractNioClientWorker worker) {
+ Collection<GridNioSession> sessions = worker.workerSessions;
+
+ int size = sessions.size();
+
+ if (size == 0)
+ return null;
+
+ int idx = ThreadLocalRandom.current().nextInt(size);
+
+ Iterator<GridNioSession> it = sessions.iterator();
+
+ int cnt = 0;
+
+ while (it.hasNext()) {
+ GridNioSession ses = it.next();
+
+ if (cnt == idx)
+ return ses;
+ }
+
+ return null;
+ }
+
+ }
+
+ /**
+ *
+ */
+ interface SessionChangeRequest {
+ GridNioSession session();
+
+ /**
+ * @return Requested change operation.
+ */
+ NioOperation operation();
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
index e4a7225..c1b60ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSession.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.util.nio;
import java.net.InetSocketAddress;
+import org.apache.ignite.IgniteCheckedException;
import org.jetbrains.annotations.Nullable;
/**
@@ -105,6 +106,11 @@ public interface GridNioSession {
public GridNioFuture<?> send(Object msg);
/**
+ * @param msg Message to be sent.
+ */
+ public void sendNoFuture(Object msg) throws IgniteCheckedException;
+
+ /**
* Gets metadata associated with specified key.
*
* @param key Key to look up.
@@ -158,10 +164,25 @@ public interface GridNioSession {
/**
* @param recoveryDesc Recovery descriptor.
*/
- public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc);
+ public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc);
+
+ /**
+ * @param recoveryDesc Recovery descriptor.
+ */
+ public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc);
/**
* @return Recovery descriptor if recovery is supported, {@code null otherwise.}
*/
- @Nullable public GridNioRecoveryDescriptor recoveryDescriptor();
+ @Nullable public GridNioRecoveryDescriptor outRecoveryDescriptor();
+
+ /**
+ * @return Recovery descriptor if recovery is supported, {@code null otherwise.}
+ */
+ @Nullable public GridNioRecoveryDescriptor inRecoveryDescriptor();
+
+ /**
+ * @param msg System message to send.
+ */
+ public void systemMessage(Object msg);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
index 0bcfe64..7424531 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
@@ -51,6 +51,12 @@ public class GridNioSessionImpl implements GridNioSession {
/** Received bytes counter. */
private volatile long bytesRcvd;
+ /** Sent bytes since last NIO sessions balancing. */
+ private volatile long bytesSent0;
+
+ /** Received bytes since last NIO sessions balancing. */
+ private volatile long bytesRcvd0;
+
/** Last send schedule timestamp. */
private volatile long sndSchedTime;
@@ -99,7 +105,7 @@ public class GridNioSessionImpl implements GridNioSession {
try {
resetSendScheduleTime();
- return chain().onSessionWrite(this, msg);
+ return chain().onSessionWrite(this, msg, true);
}
catch (IgniteCheckedException e) {
close();
@@ -109,6 +115,18 @@ public class GridNioSessionImpl implements GridNioSession {
}
/** {@inheritDoc} */
+ @Override public void sendNoFuture(Object msg) throws IgniteCheckedException {
+ try {
+ chain().onSessionWrite(this, msg, false);
+ }
+ catch (IgniteCheckedException e) {
+ close();
+
+ throw e;
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public GridNioFuture<?> resumeReads() {
try {
return chain().onResumeReads(this);
@@ -163,6 +181,28 @@ public class GridNioSessionImpl implements GridNioSession {
return bytesRcvd;
}
+ /**
+ * @return Sent bytes since last NIO sessions balancing.
+ */
+ public long bytesSent0() {
+ return bytesSent0;
+ }
+
+ /**
+ * @return Received bytes since last NIO sessions balancing.
+ */
+ public long bytesReceived0() {
+ return bytesRcvd0;
+ }
+
+ /**
+ *
+ */
+ public void reset0() {
+ bytesSent0 = 0;
+ bytesRcvd0 = 0;
+ }
+
/** {@inheritDoc} */
@Override public long createTime() {
return createTime;
@@ -240,6 +280,7 @@ public class GridNioSessionImpl implements GridNioSession {
*/
public void bytesSent(int cnt) {
bytesSent += cnt;
+ bytesSent0 += cnt;
lastSndTime = U.currentTimeMillis();
}
@@ -253,6 +294,7 @@ public class GridNioSessionImpl implements GridNioSession {
*/
public void bytesReceived(int cnt) {
bytesRcvd += cnt;
+ bytesRcvd0 += cnt;
lastRcvTime = U.currentTimeMillis();
}
@@ -296,17 +338,32 @@ public class GridNioSessionImpl implements GridNioSession {
}
/** {@inheritDoc} */
- @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
throw new UnsupportedOperationException();
}
/** {@inheritDoc} */
- @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
+ @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
return null;
}
/** {@inheritDoc} */
+ @Override public void systemMessage(Object msg) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridNioSessionImpl.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java
new file mode 100644
index 0000000..62985ff
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioWorker.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.nio;
+
+import java.util.Collection;
+import java.util.List;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+interface GridNioWorker {
+ /**
+ * @param req Change request.
+ */
+ public void offer(GridNioServer.SessionChangeRequest req);
+
+ /**
+ * @param reqs Change requests.
+ */
+ public void offer(Collection<GridNioServer.SessionChangeRequest> reqs);
+
+ /**
+ * @param ses Session.
+ * @return Session state change requests.
+ */
+ @Nullable public List<GridNioServer.SessionChangeRequest> clearSessionRequests(GridNioSession ses);
+
+ /**
+ * @param ses Session to register write interest for.
+ */
+ public void registerWrite(GridSelectorNioSessionImpl ses);
+}
[04/50] [abbrv] ignite git commit: IGNITE-500
CacheLoadingConcurrentGridStartSelfTest fails: prevent 'localUpdate'
execution while top read lock is held.
Posted by sb...@apache.org.
IGNITE-500 CacheLoadingConcurrentGridStartSelfTest fails: prevent 'localUpdate' execution while top read lock is held.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b83ec8e5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b83ec8e5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b83ec8e5
Branch: refs/heads/master
Commit: b83ec8e57c7c48f2baa4780cf3b2e46df075f3df
Parents: bc977d3
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 9 14:32:42 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 9 14:32:42 2016 +0300
----------------------------------------------------------------------
.../datastreamer/DataStreamProcessor.java | 22 +++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b83ec8e5/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 32fda87..fee4dd6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -328,6 +328,8 @@ public class DataStreamProcessor<K, V> extends GridProcessorAdapter {
if (!allowOverwrite)
cctx.topology().readLock();
+ GridDhtTopologyFuture topWaitFut = null;
+
try {
GridDhtTopologyFuture fut = cctx.topologyVersionFuture();
@@ -352,19 +354,25 @@ public class DataStreamProcessor<K, V> extends GridProcessorAdapter {
waitFut = allowOverwrite ? null : cctx.mvcc().addDataStreamerFuture(topVer);
}
- else {
- fut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
- @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
- localUpdate(nodeId, req, updater, topic);
- }
- });
- }
+ else
+ topWaitFut = fut;
}
finally {
if (!allowOverwrite)
cctx.topology().readUnlock();
}
+ if (topWaitFut != null) {
+ // Need call 'listen' after topology read lock is released.
+ topWaitFut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
+ @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
+ localUpdate(nodeId, req, updater, topic);
+ }
+ });
+
+ return;
+ }
+
if (job != null) {
try {
job.call();
[36/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
index 1b11688..87d9225 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
@@ -611,6 +611,11 @@ public class GridNearAtomicFullUpdateRequest extends GridNearAtomicAbstractUpdat
}
/** {@inheritDoc} */
+ @Override public int partition() {
+ return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
writer.setBuffer(buf);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
index 1c1addd..c3e9fbe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
@@ -125,6 +125,11 @@ public class GridNearAtomicSingleUpdateRequest extends GridNearAtomicAbstractSin
);
}
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return partId;
+ }
+
/**
* @param key Key to add.
* @param val Optional update value.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
index fa7f367..4272a4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
@@ -244,6 +244,11 @@ public class GridNearGetRequest extends GridCacheMessage implements GridCacheDep
return accessTtl;
}
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+ }
+
/**
* @param ctx Cache context.
* @throws IgniteCheckedException If failed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index a419887..bc16ff4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -1585,6 +1585,9 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
*/
@SuppressWarnings("unchecked")
protected IgniteInternalFuture asyncOp(final Callable<?> op) {
+ if (!asyncToggled)
+ return ctx.closures().callLocalSafe(op);
+
IgniteInternalFuture fail = asyncOpAcquire();
if (fail != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index d34047e..eb5e214 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -272,7 +272,7 @@ public class GridCacheDistributedQueryManager<K, V> extends GridCacheQueryManage
false,
null,
req.keyValueFilter(),
- req.partition(),
+ req.partition() == -1 ? null : req.partition(),
req.className(),
req.clause(),
req.includeMetaData(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index 60c4662..9f965d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -121,7 +121,7 @@ public class GridCacheQueryRequest extends GridCacheMessage implements GridCache
private int taskHash;
/** Partition. */
- private int part;
+ private int part = -1;
/** */
private AffinityTopologyVersion topVer;
@@ -478,8 +478,8 @@ public class GridCacheQueryRequest extends GridCacheMessage implements GridCache
/**
* @return partition.
*/
- @Nullable public Integer partition() {
- return part == -1 ? null : part;
+ public int partition() {
+ return part;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 6d21dcf..393fb1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -391,7 +391,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
/** {@inheritDoc} */
@Override public IgniteInternalFuture<Void> loadMissing(
final GridCacheContext cacheCtx,
- AffinityTopologyVersion topVer,
+ final AffinityTopologyVersion topVer,
final boolean readThrough,
boolean async,
final Collection<KeyCacheObject> keys,
@@ -472,7 +472,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
CacheObject cacheVal = cacheCtx.toCacheObject(val);
while (true) {
- GridCacheEntryEx entry = cacheCtx.cache().entryEx(key);
+ GridCacheEntryEx entry = cacheCtx.cache().entryEx(key, topVer);
try {
GridCacheVersion setVer = entry.versionedValue(cacheVal, ver, null);
@@ -1507,7 +1507,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
assert txEntry != null || readCommitted() || skipVals;
- GridCacheEntryEx e = txEntry == null ? entryEx(cacheCtx, txKey) : txEntry.cached();
+ GridCacheEntryEx e = txEntry == null ? entryEx(cacheCtx, txKey, topVer) : txEntry.cached();
if (readCommitted() || skipVals) {
cacheCtx.evicts().touch(e, topologyVersion());
@@ -1658,7 +1658,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
IgniteTxLocalAdapter.this,
/*swap*/cacheCtx.isSwapOrOffheapEnabled(),
/*unmarshal*/true,
- /**update-metrics*/true,
+ /*update-metrics*/true,
/*event*/!skipVals,
CU.subjectId(IgniteTxLocalAdapter.this, cctx),
transformClo,
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 32fda87..fee4dd6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -328,6 +328,8 @@ public class DataStreamProcessor<K, V> extends GridProcessorAdapter {
if (!allowOverwrite)
cctx.topology().readLock();
+ GridDhtTopologyFuture topWaitFut = null;
+
try {
GridDhtTopologyFuture fut = cctx.topologyVersionFuture();
@@ -352,19 +354,25 @@ public class DataStreamProcessor<K, V> extends GridProcessorAdapter {
waitFut = allowOverwrite ? null : cctx.mvcc().addDataStreamerFuture(topVer);
}
- else {
- fut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
- @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
- localUpdate(nodeId, req, updater, topic);
- }
- });
- }
+ else
+ topWaitFut = fut;
}
finally {
if (!allowOverwrite)
cctx.topology().readUnlock();
}
+ if (topWaitFut != null) {
+ // Need call 'listen' after topology read lock is released.
+ topWaitFut.listen(new IgniteInClosure<IgniteInternalFuture<AffinityTopologyVersion>>() {
+ @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> e) {
+ localUpdate(nodeId, req, updater, topic);
+ }
+ });
+
+ return;
+ }
+
if (job != null) {
try {
job.call();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
index 3405b53..4c037b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java
@@ -226,7 +226,7 @@ public class IgfsContext {
*/
public void runInIgfsThreadPool(Runnable r) {
try {
- igfsSvc.submit(r);
+ igfsSvc.execute(r);
}
catch (RejectedExecutionException ignored) {
// This exception will happen if network speed is too low and data comes faster
@@ -252,4 +252,4 @@ public class IgfsContext {
return mgr;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index e534800..4490a68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.igfs;
+import java.util.concurrent.Executor;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
@@ -36,6 +37,7 @@ import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadabl
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
@@ -325,6 +327,8 @@ public class IgfsDataManager extends IgfsManager {
IgniteInternalFuture<byte[]> fut = dataCachePrj.getAsync(key);
if (secReader != null) {
+ Executor exec = igfsCtx.kernalContext().pools().poolForPolicy(GridIoPolicy.IGFS_POOL);
+
fut = fut.chain(new CX1<IgniteInternalFuture<byte[]>, byte[]>() {
@Override public byte[] applyx(IgniteInternalFuture<byte[]> fut) throws IgniteCheckedException {
byte[] res = fut.get();
@@ -365,7 +369,7 @@ public class IgfsDataManager extends IgfsManager {
return res;
}
- });
+ }, exec);
}
else
igfsCtx.metrics().addReadBlocks(1, 0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index ab4ee85..6b23e80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -326,7 +326,7 @@ public final class IgfsImpl implements IgfsEx {
// Submit it to the thread pool immediately.
assert dualPool != null;
- dualPool.submit(batch);
+ dualPool.execute(batch);
// Spin in case another batch is currently running.
while (true) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
index 9388a8e..7cba9bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/OdbcProcessor.java
@@ -139,6 +139,7 @@ public class OdbcProcessor extends GridProcessorAdapter {
.logger(log)
.selectorCount(DFLT_SELECTOR_CNT)
.gridName(ctx.gridName())
+ .serverName("odbc")
.tcpNoDelay(DFLT_TCP_NODELAY)
.directBuffer(DFLT_TCP_DIRECT_BUF)
.byteOrder(ByteOrder.nativeOrder())
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
index 8ff15d5..5383151 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/compute/PlatformCompute.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.platform.compute;
+import java.util.concurrent.Executor;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteCompute;
import org.apache.ignite.binary.BinaryObject;
@@ -409,6 +410,11 @@ public class PlatformCompute extends PlatformAbstractTarget {
}
/** {@inheritDoc} */
+ @Override public IgniteInternalFuture chain(IgniteClosure doneCb, Executor exec) {
+ throw new UnsupportedOperationException("Chain operation is not supported.");
+ }
+
+ /** {@inheritDoc} */
@Override public Throwable error() {
return fut.error();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
index b403654..71eca65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpMemcachedNioListener.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.rest.GridRestCommand;
import org.apache.ignite.internal.processors.rest.GridRestProtocolHandler;
@@ -38,8 +37,6 @@ import org.apache.ignite.internal.util.nio.GridNioSession;
import org.apache.ignite.internal.util.typedef.C2;
import org.apache.ignite.internal.util.typedef.CIX1;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.ATOMIC_DECREMENT;
@@ -72,24 +69,16 @@ public class GridTcpMemcachedNioListener extends GridNioServerListenerAdapter<Gr
/** Handler. */
private final GridRestProtocolHandler hnd;
- /** JDK marshaller. */
- private final Marshaller jdkMarshaller = new JdkMarshaller();
-
- /** Context. */
- private final GridKernalContext ctx;
-
/**
* Creates listener which will convert incoming tcp packets to rest requests and forward them to
* a given rest handler.
*
* @param log Logger to use.
* @param hnd Rest handler.
- * @param ctx Context.
*/
- public GridTcpMemcachedNioListener(IgniteLogger log, GridRestProtocolHandler hnd, GridKernalContext ctx) {
+ public GridTcpMemcachedNioListener(IgniteLogger log, GridRestProtocolHandler hnd) {
this.log = log;
this.hnd = hnd;
- this.ctx = ctx;
}
/** {@inheritDoc} */
@@ -462,4 +451,4 @@ public class GridTcpMemcachedNioListener extends GridNioServerListenerAdapter<Gr
return new GridTuple3<>(cmd, quiet, retKey);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
index 1c1c6dc..3ba6d8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
@@ -145,7 +145,7 @@ public class GridTcpRestNioListener extends GridNioServerListenerAdapter<GridCli
*/
public GridTcpRestNioListener(IgniteLogger log, GridTcpRestProtocol proto, GridRestProtocolHandler hnd,
GridKernalContext ctx) {
- memcachedLsnr = new GridTcpMemcachedNioListener(log, hnd, ctx);
+ memcachedLsnr = new GridTcpMemcachedNioListener(log, hnd);
redisLsnr = new GridRedisNioListener(log, hnd, ctx);
this.log = log;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
index 6338fcc..2a002a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
@@ -257,6 +257,7 @@ public class GridTcpRestProtocol extends GridRestProtocolAdapter {
.logger(log)
.selectorCount(cfg.getSelectorCount())
.gridName(ctx.gridName())
+ .serverName("tcp-rest")
.tcpNoDelay(cfg.isNoDelay())
.directBuffer(cfg.isDirectBuffer())
.byteOrder(ByteOrder.nativeOrder())
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6c26363..aeb3ef4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1131,7 +1131,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
// Start service in its own thread.
final ExecutorService exe = svcCtx.executor();
- exe.submit(new Runnable() {
+ exe.execute(new Runnable() {
@Override public void run() {
try {
svc.execute(svcCtx);
@@ -1394,7 +1394,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
return;
try {
- depExe.submit(new BusyRunnable() {
+ depExe.execute(new BusyRunnable() {
@Override public void run0() {
onSystemCacheUpdated(deps);
}
@@ -1587,7 +1587,7 @@ public class GridServiceProcessor extends GridProcessorAdapter {
else
topVer = new AffinityTopologyVersion(((DiscoveryEvent)evt).topologyVersion(), 0);
- depExe.submit(new BusyRunnable() {
+ depExe.execute(new BusyRunnable() {
@Override public void run0() {
ClusterNode oldest = CU.oldestAliveCacheServerNode(cache.context().shared(), topVer);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index e1937bb..3dfb3c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -153,7 +153,6 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.binary.BinaryRawReader;
import org.apache.ignite.binary.BinaryRawWriter;
import org.apache.ignite.cluster.ClusterGroupEmptyException;
@@ -506,10 +505,27 @@ public abstract class IgniteUtils {
}
};
- /**
- * Initializes enterprise check.
+ /** */
+ private static final boolean assertionsEnabled;
+
+ /*
+ *
*/
static {
+ boolean assertionsEnabled0 = true;
+
+ try {
+ assert false;
+
+ assertionsEnabled0 = false;
+ }
+ catch (AssertionError ignored) {
+ assertionsEnabled0 = true;
+ }
+ finally {
+ assertionsEnabled = assertionsEnabled0;
+ }
+
String osName = System.getProperty("os.name");
String osLow = osName.toLowerCase();
@@ -1284,6 +1300,27 @@ public abstract class IgniteUtils {
}
/**
+ * @param threadId Thread ID.
+ * @param sb Builder.
+ */
+ public static void printStackTrace(long threadId, GridStringBuilder sb) {
+ ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+
+ ThreadInfo threadInfo = mxBean.getThreadInfo(threadId, Integer.MAX_VALUE);
+
+ printThreadInfo(threadInfo, sb, Collections.<Long>emptySet());
+ }
+
+ /**
+ * @return {@code true} if there is java level deadlock.
+ */
+ public static boolean deadlockPresent() {
+ ThreadMXBean mxBean = ManagementFactory.getThreadMXBean();
+
+ return !F.isEmpty(mxBean.findDeadlockedThreads());
+ }
+
+ /**
* Prints single thread info to a buffer.
*
* @param threadInfo Thread info.
@@ -6141,6 +6178,13 @@ public abstract class IgniteUtils {
}
/**
+ * @return {@code True} if assertions enabled.
+ */
+ public static boolean assertionsEnabled() {
+ return assertionsEnabled;
+ }
+
+ /**
* Gets OS JDK string.
*
* @return OS JDK string.
@@ -8337,6 +8381,18 @@ public abstract class IgniteUtils {
}
/**
+ * Gets absolute value for long. If argument is {@link Long#MIN_VALUE}, then {@code 0} is returned.
+ *
+ * @param i Argument.
+ * @return Absolute value.
+ */
+ public static long safeAbs(long i) {
+ i = Math.abs(i);
+
+ return i < 0 ? 0 : i;
+ }
+
+ /**
* Gets wrapper class for a primitive type.
*
* @param cls Class. If {@code null}, method is no-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
new file mode 100644
index 0000000..e9ec74b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -0,0 +1,667 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Striped executor.
+ */
+public class StripedExecutor implements ExecutorService {
+ /** Stripes. */
+ private final Stripe[] stripes;
+
+ /** For starvation checks. */
+ private final long[] completedCntrs;
+
+ /** */
+ private final IgniteLogger log;
+
+ /**
+ * Constructor.
+ *
+ * @param cnt Count.
+ */
+ public StripedExecutor(int cnt, String gridName, String poolName, final IgniteLogger log) {
+ A.ensure(cnt > 0, "cnt > 0");
+
+ boolean success = false;
+
+ stripes = new Stripe[cnt];
+
+ completedCntrs = new long[cnt];
+
+ Arrays.fill(completedCntrs, -1);
+
+ this.log = log;
+
+ try {
+ for (int i = 0; i < cnt; i++) {
+ stripes[i] = new StripeConcurrentQueue(
+ gridName,
+ poolName,
+ i,
+ log);
+
+ stripes[i].start();
+ }
+
+ success = true;
+ }
+ catch (Error | RuntimeException e) {
+ U.error(log, "Failed to initialize striped pool.", e);
+
+ throw e;
+ }
+ finally {
+ if (!success) {
+ for (Stripe stripe : stripes) {
+ if (stripe != null)
+ stripe.signalStop();
+ }
+
+ for (Stripe stripe : stripes) {
+ if (stripe != null)
+ stripe.awaitStop();
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks starvation in striped pool. Maybe too verbose
+ * but this is needed to faster debug possible issues.
+ */
+ public void checkStarvation() {
+ for (int i = 0; i < stripes.length; i++) {
+ Stripe stripe = stripes[i];
+
+ long completedCnt = stripe.completedCnt;
+
+ boolean active = stripe.active;
+
+ if (completedCntrs[i] != -1 &&
+ completedCntrs[i] == completedCnt &&
+ active) {
+ boolean deadlockPresent = U.deadlockPresent();
+
+ GridStringBuilder sb = new GridStringBuilder();
+
+ sb.a(">>> Possible starvation in striped pool: ")
+ .a(stripe.thread.getName()).a(U.nl())
+ .a(stripe.queueToString()).a(U.nl())
+ .a("deadlock: ").a(deadlockPresent).a(U.nl())
+ .a("completed: ").a(completedCnt).a(U.nl());
+
+ U.printStackTrace(
+ stripe.thread.getId(),
+ sb);
+
+ String msg = sb.toString();
+
+ U.warn(log, msg);
+ }
+
+ if (active || completedCnt > 0)
+ completedCntrs[i] = completedCnt;
+ }
+ }
+
+ /**
+ * @return Stripes count.
+ */
+ public int stripes() {
+ return stripes.length;
+ }
+
+ /**
+ * Execute command.
+ *
+ * @param idx Index.
+ * @param cmd Command.
+ */
+ public void execute(int idx, Runnable cmd) {
+ if (idx == -1)
+ execute(cmd);
+ else {
+ assert idx >= 0 : idx;
+
+ stripes[idx % stripes.length].execute(cmd);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void shutdown() {
+ signalStop();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute(@NotNull Runnable cmd) {
+ stripes[ThreadLocalRandom.current().nextInt(stripes.length)].execute(cmd);
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * @return Empty list (always).
+ */
+ @NotNull @Override public List<Runnable> shutdownNow() {
+ signalStop();
+
+ return Collections.emptyList();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean awaitTermination(
+ long timeout,
+ @NotNull TimeUnit unit
+ ) throws InterruptedException {
+ awaitStop();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isShutdown() {
+ for (Stripe stripe : stripes) {
+ if (stripe != null && stripe.stopping)
+ return true;
+ }
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isTerminated() {
+ for (Stripe stripe : stripes) {
+ if (stripe.thread.getState() != Thread.State.TERMINATED)
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Stops executor.
+ */
+ public void stop() {
+ signalStop();
+
+ awaitStop();
+ }
+
+ /**
+ * Signals all stripes.
+ */
+ private void signalStop() {
+ for (Stripe stripe : stripes)
+ stripe.signalStop();
+ }
+
+ /**
+ * @throws IgniteInterruptedException If interrupted.
+ */
+ private void awaitStop() throws IgniteInterruptedException {
+ for (Stripe stripe : stripes)
+ stripe.awaitStop();
+ }
+
+ /**
+ * @return Return total queue size of all stripes.
+ */
+ public int queueSize() {
+ int size = 0;
+
+ for (Stripe stripe : stripes)
+ size += stripe.queueSize();
+
+ return size;
+ }
+
+ /**
+ * @return Completed tasks count.
+ */
+ public long completedTasks() {
+ long cnt = 0;
+
+ for (Stripe stripe : stripes)
+ cnt += stripe.completedCnt;
+
+ return cnt;
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @NotNull @Override public <T> Future<T> submit(
+ @NotNull Runnable task,
+ T res
+ ) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @NotNull @Override public Future<?> submit(@NotNull Runnable task) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @NotNull @Override public <T> Future<T> submit(@NotNull Callable<T> task) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @NotNull @Override public <T> List<Future<T>> invokeAll(@NotNull Collection<? extends Callable<T>> tasks)
+ throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @NotNull @Override public <T> List<Future<T>> invokeAll(
+ @NotNull Collection<? extends Callable<T>> tasks,
+ long timeout,
+ @NotNull TimeUnit unit
+ ) throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @NotNull @Override public <T> T invokeAny(@NotNull Collection<? extends Callable<T>> tasks)
+ throws InterruptedException, ExecutionException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Operation not supported.
+ */
+ @Override public <T> T invokeAny(
+ @NotNull Collection<? extends Callable<T>> tasks,
+ long timeout,
+ @NotNull TimeUnit unit
+ ) throws InterruptedException, ExecutionException, TimeoutException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StripedExecutor.class, this);
+ }
+
+ /**
+ * Stripe.
+ */
+ private static abstract class Stripe implements Runnable {
+ /** */
+ private final String gridName;
+
+ /** */
+ private final String poolName;
+
+ /** */
+ private final int idx;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** Stopping flag. */
+ private volatile boolean stopping;
+
+ /** */
+ private volatile long completedCnt;
+
+ /** */
+ private volatile boolean active;
+
+ /** Thread executing the loop. */
+ protected Thread thread;
+
+ /**
+ * @param gridName Grid name.
+ * @param poolName Pool name.
+ * @param idx Stripe index.
+ * @param log Logger.
+ */
+ public Stripe(
+ String gridName,
+ String poolName,
+ int idx,
+ IgniteLogger log
+ ) {
+ this.gridName = gridName;
+ this.poolName = poolName;
+ this.idx = idx;
+ this.log = log;
+ }
+
+ /**
+ * Starts the stripe.
+ */
+ void start() {
+ thread = new IgniteThread(gridName, poolName + "-stripe-" + idx, this);
+
+ thread.start();
+ }
+
+ /**
+ * Stop the stripe.
+ */
+ void signalStop() {
+ stopping = true;
+
+ U.interrupt(thread);
+ }
+
+ /**
+ * Await thread stop.
+ */
+ void awaitStop() {
+ try {
+ if (thread != null)
+ thread.join();
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ while (!stopping) {
+ Runnable cmd;
+
+ try {
+ cmd = take();
+
+ if (cmd != null) {
+ active = true;
+
+ try {
+ cmd.run();
+ }
+ finally {
+ active = false;
+ completedCnt++;
+ }
+ }
+ }
+ catch (InterruptedException e) {
+ stopping = true;
+
+ Thread.currentThread().interrupt();
+
+ return;
+ }
+ catch (Throwable e) {
+ U.error(log, "Failed to execute runnable.", e);
+ }
+ }
+ }
+
+ /**
+ * Execute the command.
+ *
+ * @param cmd Command.
+ */
+ abstract void execute(Runnable cmd);
+
+ /**
+ * @return Next runnable.
+ * @throws InterruptedException If interrupted.
+ */
+ abstract Runnable take() throws InterruptedException;
+
+ /**
+ * @return Queue size.
+ */
+ abstract int queueSize();
+
+ /**
+ * @return Stripe's queue to string presentation.
+ */
+ abstract String queueToString();
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Stripe.class, this);
+ }
+ }
+
+ /**
+ * Stripe.
+ */
+ private static class StripeConcurrentQueue extends Stripe {
+ /** Queue. */
+ private final Queue<Runnable> queue = new ConcurrentLinkedQueue<>();
+
+ /** */
+ private volatile boolean parked;
+
+ /**
+ * @param gridName Grid name.
+ * @param poolName Pool name.
+ * @param idx Stripe index.
+ * @param log Logger.
+ */
+ public StripeConcurrentQueue(
+ String gridName,
+ String poolName,
+ int idx,
+ IgniteLogger log
+ ) {
+ super(gridName,
+ poolName,
+ idx,
+ log);
+ }
+
+ /** {@inheritDoc} */
+ @Override Runnable take() throws InterruptedException {
+ Runnable r;
+
+ for (int i = 0; i < 2048; i++) {
+ r = queue.poll();
+
+ if (r != null)
+ return r;
+ }
+
+ parked = true;
+
+ try {
+ for (;;) {
+ r = queue.poll();
+
+ if (r != null)
+ return r;
+
+ LockSupport.park();
+
+ if (Thread.interrupted())
+ throw new InterruptedException();
+ }
+ }
+ finally {
+ parked = false;
+ }
+ }
+
+ /** {@inheritDoc} */
+ void execute(Runnable cmd) {
+ queue.add(cmd);
+
+ if (parked)
+ LockSupport.unpark(thread);
+ }
+
+ /** {@inheritDoc} */
+ @Override String queueToString() {
+ return String.valueOf(queue);
+ }
+
+ /** {@inheritDoc} */
+ @Override int queueSize() {
+ return queue.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StripeConcurrentQueue.class, this, super.toString());
+ }
+ }
+
+ /**
+ * Stripe.
+ */
+ private static class StripeConcurrentQueueNoPark extends Stripe {
+ /** Queue. */
+ private final Queue<Runnable> queue = new ConcurrentLinkedQueue<>();
+
+ /**
+ * @param gridName Grid name.
+ * @param poolName Pool name.
+ * @param idx Stripe index.
+ * @param log Logger.
+ */
+ public StripeConcurrentQueueNoPark(
+ String gridName,
+ String poolName,
+ int idx,
+ IgniteLogger log
+ ) {
+ super(gridName,
+ poolName,
+ idx,
+ log);
+ }
+
+ /** {@inheritDoc} */
+ @Override Runnable take() {
+ for (;;) {
+ Runnable r = queue.poll();
+
+ if (r != null)
+ return r;
+ }
+ }
+
+ /** {@inheritDoc} */
+ void execute(Runnable cmd) {
+ queue.add(cmd);
+ }
+
+ /** {@inheritDoc} */
+ @Override int queueSize() {
+ return queue.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override String queueToString() {
+ return String.valueOf(queue);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StripeConcurrentQueueNoPark.class, this, super.toString());
+ }
+ }
+
+ /**
+ * Stripe.
+ */
+ private static class StripeConcurrentBlockingQueue extends Stripe {
+ /** Queue. */
+ private final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+
+ /**
+ * @param gridName Grid name.
+ * @param poolName Pool name.
+ * @param idx Stripe index.
+ * @param log Logger.
+ */
+ public StripeConcurrentBlockingQueue(
+ String gridName,
+ String poolName,
+ int idx,
+ IgniteLogger log
+ ) {
+ super(gridName,
+ poolName,
+ idx,
+ log);
+ }
+
+ /** {@inheritDoc} */
+ @Override Runnable take() throws InterruptedException {
+ return queue.take();
+ }
+
+ /** {@inheritDoc} */
+ void execute(Runnable cmd) {
+ queue.add(cmd);
+ }
+
+ /** {@inheritDoc} */
+ @Override int queueSize() {
+ return queue.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override String queueToString() {
+ return String.valueOf(queue);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StripeConcurrentBlockingQueue.class, this, super.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
index 6baedbd..dc63adc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFinishedFuture.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.util.future;
+import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -152,6 +153,29 @@ public class GridFinishedFuture<T> implements IgniteInternalFuture<T> {
}
/** {@inheritDoc} */
+ @Override public <T1> IgniteInternalFuture<T1> chain(final IgniteClosure<? super IgniteInternalFuture<T>, T1> doneCb, Executor exec) {
+ final GridFutureAdapter<T1> fut = new GridFutureAdapter<>();
+
+ exec.execute(new Runnable() {
+ @Override public void run() {
+ try {
+ fut.onDone(doneCb.apply(GridFinishedFuture.this));
+ }
+ catch (GridClosureException e) {
+ fut.onDone(e.unwrap());
+ }
+ catch (RuntimeException | Error e) {
+ fut.onDone(e);
+
+ throw e;
+ }
+ }
+ });
+
+ return fut;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridFinishedFuture.class, this);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 2cd534e..c8d85cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.util.future;
import java.util.Arrays;
+import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.AbstractQueuedSynchronizer;
import org.apache.ignite.IgniteCheckedException;
@@ -229,7 +230,13 @@ public class GridFutureAdapter<R> extends AbstractQueuedSynchronizer implements
/** {@inheritDoc} */
@Override public <T> IgniteInternalFuture<T> chain(final IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb) {
- return new ChainFuture<>(this, doneCb);
+ return new ChainFuture<>(this, doneCb, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> IgniteInternalFuture<T> chain(final IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb,
+ Executor exec) {
+ return new ChainFuture<>(this, doneCb, exec);
}
/**
@@ -487,15 +494,17 @@ public class GridFutureAdapter<R> extends AbstractQueuedSynchronizer implements
/**
* @param fut Future.
* @param doneCb Closure.
+ * @param cbExec Optional executor to run callback.
*/
ChainFuture(
GridFutureAdapter<R> fut,
- IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb
+ IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb,
+ @Nullable Executor cbExec
) {
this.fut = fut;
this.doneCb = doneCb;
- fut.listen(new GridFutureChainListener<>(this, doneCb));
+ fut.listen(new GridFutureChainListener<>(this, doneCb, cbExec));
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
index 947b2ad..367f5d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureChainListener.java
@@ -17,15 +17,17 @@
package org.apache.ignite.internal.util.future;
+import java.util.concurrent.Executor;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.lang.GridClosureException;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteInClosure;
+import org.jetbrains.annotations.Nullable;
/**
* Future listener to fill chained future with converted result of the source future.
*/
-public class GridFutureChainListener<T, R> implements IgniteInClosure<IgniteInternalFuture<T>> {
+class GridFutureChainListener<T, R> implements IgniteInClosure<IgniteInternalFuture<T>> {
/** */
private static final long serialVersionUID = 0L;
@@ -35,21 +37,43 @@ public class GridFutureChainListener<T, R> implements IgniteInClosure<IgniteInte
/** Done callback. */
private final IgniteClosure<? super IgniteInternalFuture<T>, R> doneCb;
+ /** */
+ private Executor cbExec;
+
/**
* Constructs chain listener.
+ *
* @param fut Target future.
* @param doneCb Done callback.
+ * @param cbExec Optional executor to run callback.
*/
public GridFutureChainListener(
GridFutureAdapter<R> fut,
- IgniteClosure<? super IgniteInternalFuture<T>, R> doneCb
+ IgniteClosure<? super IgniteInternalFuture<T>, R> doneCb,
+ @Nullable Executor cbExec
) {
this.fut = fut;
this.doneCb = doneCb;
+ this.cbExec = cbExec;
}
/** {@inheritDoc} */
- @Override public void apply(IgniteInternalFuture<T> t) {
+ @Override public void apply(final IgniteInternalFuture<T> t) {
+ if (cbExec != null) {
+ cbExec.execute(new Runnable() {
+ @Override public void run() {
+ applyCallback(t);
+ }
+ });
+ }
+ else
+ applyCallback(t);
+ }
+
+ /**
+ * @param t Target future.
+ */
+ private void applyCallback(IgniteInternalFuture<T> t) {
try {
fut.onDone(doneCb.apply(t));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
index 6820dc7..d108b56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/ipc/IpcToNioAdapter.java
@@ -201,7 +201,7 @@ public class IpcToNioAdapter<T> {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) {
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) {
assert ses == IpcToNioAdapter.this.ses;
return send((Message)msg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
index 9b014ec..f2ab932 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridAbstractCommunicationClient.java
@@ -35,14 +35,24 @@ public abstract class GridAbstractCommunicationClient implements GridCommunicati
/** Metrics listener. */
protected final GridNioMetricsListener metricsLsnr;
+ /** */
+ private final int connIdx;
+
/**
+ * @param connIdx Connection index.
* @param metricsLsnr Metrics listener.
*/
- protected GridAbstractCommunicationClient(@Nullable GridNioMetricsListener metricsLsnr) {
+ protected GridAbstractCommunicationClient(int connIdx, @Nullable GridNioMetricsListener metricsLsnr) {
+ this.connIdx = connIdx;
this.metricsLsnr = metricsLsnr;
}
/** {@inheritDoc} */
+ @Override public int connectionIndex() {
+ return connIdx;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean close() {
return reserves.compareAndSet(0, -1);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
index 0de54e9..71b2c24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridCommunicationClient.java
@@ -96,15 +96,20 @@ public interface GridCommunicationClient {
/**
* @param nodeId Remote node ID. Provided only for sync clients.
* @param msg Message to send.
- * @param closure Ack closure.
+ * @param c Ack closure.
* @throws IgniteCheckedException If failed.
* @return {@code True} if should try to resend message.
*/
- public boolean sendMessage(@Nullable UUID nodeId, Message msg, @Nullable IgniteInClosure<IgniteException> closure)
+ public boolean sendMessage(@Nullable UUID nodeId, Message msg, @Nullable IgniteInClosure<IgniteException> c)
throws IgniteCheckedException;
/**
* @return {@code True} if send is asynchronous.
*/
public boolean async();
+
+ /**
+ * @return Connection index.
+ */
+ public int connectionIndex();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
index 213fd8d..7987d3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridConnectionBytesVerifyFilter.java
@@ -62,13 +62,20 @@ public class GridConnectionBytesVerifyFilter extends GridNioFilterAdapter {
}
/** {@inheritDoc} */
- @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+ @Override public void onExceptionCaught(
+ GridNioSession ses,
+ IgniteCheckedException ex
+ ) throws IgniteCheckedException {
proceedExceptionCaught(ses, ex);
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
- return proceedSessionWrite(ses, msg);
+ @Override public GridNioFuture<?> onSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException {
+ return proceedSessionWrite(ses, msg, fut);
}
/** {@inheritDoc} */
@@ -137,4 +144,4 @@ public class GridConnectionBytesVerifyFilter extends GridNioFilterAdapter {
@Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
proceedSessionWriteTimeout(ses);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
index 9925d2e..40c87cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioAsyncNotifyFilter.java
@@ -107,8 +107,12 @@ public class GridNioAsyncNotifyFilter extends GridNioFilterAdapter {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
- return proceedSessionWrite(ses, msg);
+ @Override public GridNioFuture<?> onSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException {
+ return proceedSessionWrite(ses, msg, fut);
}
/** {@inheritDoc} */
@@ -139,4 +143,4 @@ public class GridNioAsyncNotifyFilter extends GridNioFilterAdapter {
"originalEx=" + ex + ", ex=" + e + ']');
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
index 7083ccf..343e625 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioCodecFilter.java
@@ -71,20 +71,27 @@ public class GridNioCodecFilter extends GridNioFilterAdapter {
}
/** {@inheritDoc} */
- @Override public void onExceptionCaught(GridNioSession ses, IgniteCheckedException ex) throws IgniteCheckedException {
+ @Override public void onExceptionCaught(
+ GridNioSession ses,
+ IgniteCheckedException ex
+ ) throws IgniteCheckedException {
proceedExceptionCaught(ses, ex);
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ @Override public GridNioFuture<?> onSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException {
// No encoding needed in direct mode.
if (directMode)
- return proceedSessionWrite(ses, msg);
+ return proceedSessionWrite(ses, msg, fut);
try {
ByteBuffer res = parser.encode(ses, msg);
- return proceedSessionWrite(ses, res);
+ return proceedSessionWrite(ses, res, fut);
}
catch (IOException e) {
throw new GridNioException(e);
@@ -137,4 +144,4 @@ public class GridNioCodecFilter extends GridNioFilterAdapter {
@Override public void onSessionWriteTimeout(GridNioSession ses) throws IgniteCheckedException {
proceedSessionWriteTimeout(ses);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
index 5f88b1f..f7928c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilter.java
@@ -105,10 +105,15 @@ public interface GridNioFilter {
*
* @param ses Session instance.
* @param msg Message to send.
- * @return Write future.
+ * @param fut {@code True} if write future should be created.
+ * @return Write future or {@code null}.
* @throws IgniteCheckedException If filter is not in chain or GridNioException occurred in the underlying filter.
*/
- public GridNioFuture<?> proceedSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException;
+ public GridNioFuture<?> proceedSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException;
/**
* Forwards session close request to the next logical filter in filter chain.
@@ -149,10 +154,11 @@ public interface GridNioFilter {
*
* @param ses Session on which message should be written.
* @param msg Message being written.
- * @return Write future.
+ * @param fut {@code True} if write future should be created.
+ * @return Write future or {@code null}.
* @throws GridNioException If GridNioException occurred while handling event.
*/
- public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException;
+ public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut) throws IgniteCheckedException;
/**
* Invoked when a new messages received.
@@ -241,4 +247,4 @@ public interface GridNioFilter {
* @throws IgniteCheckedException If filter is not in chain or GridNioException occurred in the underlying filter.
*/
public GridNioFuture<?> onResumeReads(GridNioSession ses) throws IgniteCheckedException;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
index 18ab1b2..58ddae5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterAdapter.java
@@ -108,10 +108,14 @@ public abstract class GridNioFilterAdapter implements GridNioFilter {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> proceedSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
+ @Override public GridNioFuture<?> proceedSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException {
checkNext();
- return nextFilter.onSessionWrite(ses, msg);
+ return nextFilter.onSessionWrite(ses, msg, fut);
}
/** {@inheritDoc} */
@@ -180,4 +184,4 @@ public abstract class GridNioFilterAdapter implements GridNioFilter {
throw new GridNioException("Failed to proceed with filter call since previous filter is not set " +
"(do you use filter outside the filter chain?): " + getClass().getName());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
index a3a74e3..8cc690b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFilterChain.java
@@ -181,8 +181,12 @@ public class GridNioFilterChain<T> extends GridNioFilterAdapter {
* @return Send future.
* @throws IgniteCheckedException If IgniteCheckedException occurred while handling event.
*/
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg) throws IgniteCheckedException {
- return tail.onSessionWrite(ses, msg);
+ @Override public GridNioFuture<?> onSessionWrite(
+ GridNioSession ses,
+ Object msg,
+ boolean fut
+ ) throws IgniteCheckedException {
+ return tail.onSessionWrite(ses, msg, fut);
}
/**
@@ -255,9 +259,9 @@ public class GridNioFilterChain<T> extends GridNioFilterAdapter {
}
/** {@inheritDoc} */
- @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg)
+ @Override public GridNioFuture<?> onSessionWrite(GridNioSession ses, Object msg, boolean fut)
throws IgniteCheckedException {
- return proceedSessionWrite(ses, msg);
+ return proceedSessionWrite(ses, msg, fut);
}
/** {@inheritDoc} */
@@ -290,4 +294,4 @@ public class GridNioFilterChain<T> extends GridNioFilterAdapter {
return proceedResumeReads(ses);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
index b02acc8..6c0c9c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioFuture.java
@@ -45,9 +45,9 @@ public interface GridNioFuture<R> extends IgniteInternalFuture<R> {
/**
* Sets ack closure which will be applied when ack received.
*
- * @param closure Ack closure.
+ * @param c Ack closure.
*/
- public void ackClosure(IgniteInClosure<IgniteException> closure);
+ public void ackClosure(IgniteInClosure<IgniteException> c);
/**
* The method will be called when ack received.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
index 35480ac..6258c13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
@@ -35,8 +35,8 @@ public class GridNioRecoveryDescriptor {
/** Number of acknowledged messages. */
private long acked;
- /** Unacknowledged message futures. */
- private final ArrayDeque<GridNioFuture<?>> msgFuts;
+ /** Unacknowledged messages. */
+ private final ArrayDeque<SessionWriteRequest> msgReqs;
/** Number of messages to resend. */
private int resendCnt;
@@ -77,23 +77,40 @@ public class GridNioRecoveryDescriptor {
/** Number of descriptor reservations (for info purposes). */
private int reserveCnt;
+ /** */
+ private final boolean pairedConnections;
+
/**
+ * @param pairedConnections {@code True} if in/out connections pair is used for communication with node.
* @param queueLimit Maximum size of unacknowledged messages queue.
* @param node Node.
* @param log Logger.
*/
- public GridNioRecoveryDescriptor(int queueLimit, ClusterNode node, IgniteLogger log) {
+ public GridNioRecoveryDescriptor(
+ boolean pairedConnections,
+ int queueLimit,
+ ClusterNode node,
+ IgniteLogger log
+ ) {
assert !node.isLocal() : node;
assert queueLimit > 0;
- msgFuts = new ArrayDeque<>(queueLimit);
+ msgReqs = new ArrayDeque<>(queueLimit);
+ this.pairedConnections = pairedConnections;
this.queueLimit = queueLimit;
this.node = node;
this.log = log;
}
/**
+ * @return {@code True} if in/out connections pair is used for communication with node.
+ */
+ public boolean pairedConnections() {
+ return pairedConnections;
+ }
+
+ /**
* @return Connect count.
*/
public long incrementConnectCount() {
@@ -154,19 +171,19 @@ public class GridNioRecoveryDescriptor {
}
/**
- * @param fut NIO future.
+ * @param req Write request.
* @return {@code False} if queue limit is exceeded.
*/
- public boolean add(GridNioFuture<?> fut) {
- assert fut != null;
+ public boolean add(SessionWriteRequest req) {
+ assert req != null;
- if (!fut.skipRecovery()) {
+ if (!req.skipRecovery()) {
if (resendCnt == 0) {
- msgFuts.addLast(fut);
+ msgReqs.addLast(req);
sentCnt++;
- return msgFuts.size() < queueLimit;
+ return msgReqs.size() < queueLimit;
}
else
resendCnt--;
@@ -181,21 +198,19 @@ public class GridNioRecoveryDescriptor {
public void ackReceived(long rcvCnt) {
if (log.isDebugEnabled())
log.debug("Handle acknowledgment [acked=" + acked + ", rcvCnt=" + rcvCnt +
- ", msgFuts=" + msgFuts.size() + ']');
+ ", msgReqs=" + msgReqs.size() + ']');
while (acked < rcvCnt) {
- GridNioFuture<?> fut = msgFuts.pollFirst();
+ SessionWriteRequest req = msgReqs.pollFirst();
- assert fut != null : "Missed message future [rcvCnt=" + rcvCnt +
+ assert req != null : "Missed message [rcvCnt=" + rcvCnt +
", acked=" + acked +
", desc=" + this + ']';
- assert fut.isDone() : fut;
-
- if (fut.ackClosure() != null)
- fut.ackClosure().apply(null);
+ if (req.ackClosure() != null)
+ req.ackClosure().apply(null);
- fut.onAckReceived();
+ req.onAckReceived();
acked++;
}
@@ -214,7 +229,7 @@ public class GridNioRecoveryDescriptor {
* @return {@code False} if descriptor is reserved.
*/
public boolean onNodeLeft() {
- GridNioFuture<?>[] futs = null;
+ SessionWriteRequest[] reqs = null;
synchronized (this) {
nodeLeft = true;
@@ -222,24 +237,24 @@ public class GridNioRecoveryDescriptor {
if (reserved)
return false;
- if (!msgFuts.isEmpty()) {
- futs = msgFuts.toArray(new GridNioFuture<?>[msgFuts.size()]);
+ if (!msgReqs.isEmpty()) {
+ reqs = msgReqs.toArray(new SessionWriteRequest[msgReqs.size()]);
- msgFuts.clear();
+ msgReqs.clear();
}
}
- if (futs != null)
- completeOnNodeLeft(futs);
+ if (reqs != null)
+ notifyOnNodeLeft(reqs);
return true;
}
/**
- * @return Message futures for unacknowledged messages.
+ * @return Requests for unacknowledged messages.
*/
- public Deque<GridNioFuture<?>> messagesFutures() {
- return msgFuts;
+ public Deque<SessionWriteRequest> messagesRequests() {
+ return msgReqs;
}
/**
@@ -277,14 +292,14 @@ public class GridNioRecoveryDescriptor {
if (!nodeLeft)
ackReceived(rcvCnt);
- resendCnt = msgFuts.size();
+ resendCnt = msgReqs.size();
}
}
/**
*
*/
- public void connected() {
+ public void onConnected() {
synchronized (this) {
assert reserved : this;
assert !connected : this;
@@ -306,10 +321,37 @@ public class GridNioRecoveryDescriptor {
}
/**
+ * @return Connected flag.
+ */
+ public boolean connected() {
+ synchronized (this) {
+ return connected;
+ }
+ }
+
+ /**
+ * @return Reserved flag.
+ */
+ public boolean reserved() {
+ synchronized (this) {
+ return reserved;
+ }
+ }
+
+ /**
+ * @return Current handshake index.
+ */
+ public Long handshakeIndex() {
+ synchronized (this) {
+ return handshakeReq != null ? handshakeReq.get1() : null;
+ }
+ }
+
+ /**
*
*/
public void release() {
- GridNioFuture<?>[] futs = null;
+ SessionWriteRequest[] futs = null;
synchronized (this) {
connected = false;
@@ -329,15 +371,15 @@ public class GridNioRecoveryDescriptor {
notifyAll();
}
- if (nodeLeft && !msgFuts.isEmpty()) {
- futs = msgFuts.toArray(new GridNioFuture<?>[msgFuts.size()]);
+ if (nodeLeft && !msgReqs.isEmpty()) {
+ futs = msgReqs.toArray(new SessionWriteRequest[msgReqs.size()]);
- msgFuts.clear();
+ msgReqs.clear();
}
}
if (futs != null)
- completeOnNodeLeft(futs);
+ notifyOnNodeLeft(futs);
}
/**
@@ -398,16 +440,16 @@ public class GridNioRecoveryDescriptor {
}
/**
- * @param futs Futures to complete.
+ * @param reqs Requests to notify about error.
*/
- private void completeOnNodeLeft(GridNioFuture<?>[] futs) {
- for (GridNioFuture<?> msg : futs) {
- IOException e = new IOException("Failed to send message, node has left: " + node.id());
+ private void notifyOnNodeLeft(SessionWriteRequest[] reqs) {
+ IOException e = new IOException("Failed to send message, node has left: " + node.id());
- ((GridNioFutureImpl)msg).onDone(e);
+ for (SessionWriteRequest req : reqs) {
+ req.onError(e);
- if (msg.ackClosure() != null)
- msg.ackClosure().apply(new IgniteException(e));
+ if (req.ackClosure() != null)
+ req.ackClosure().apply(new IgniteException(e));
}
}
[10/50] [abbrv] ignite git commit: Merge remote-tracking branch
'origin/ignite-1.7.4' into ignite-1.7.4
Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/ignite-1.7.4' into ignite-1.7.4
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ceb60d27
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ceb60d27
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ceb60d27
Branch: refs/heads/master
Commit: ceb60d27d38782d82ec5275a56a6f778511111b6
Parents: f087b66 6616a87
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Fri Dec 9 17:46:21 2016 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Fri Dec 9 17:46:21 2016 +0300
----------------------------------------------------------------------
.../internal/binary/BinaryClassDescriptor.java | 12 ++--
.../ignite/internal/binary/BinaryUtils.java | 10 ++-
.../binary/builder/BinaryObjectBuilderImpl.java | 11 +++-
.../platform/PlatformContextImpl.java | 2 +-
.../platform/utils/PlatformUtils.java | 28 +++++++++
.../binary/BinaryMarshallerSelfTest.java | 66 ++++++++++++++++++++
6 files changed, 119 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
[08/50] [abbrv] ignite git commit: Merge remote-tracking branch
'upstream/ignite-1.7.4' into ignite-1.7.4
Posted by sb...@apache.org.
Merge remote-tracking branch 'upstream/ignite-1.7.4' into ignite-1.7.4
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6616a878
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6616a878
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6616a878
Branch: refs/heads/master
Commit: 6616a878598b8ed74b71558290a22b266a8ac03e
Parents: c143bc1 6fd8bf6
Author: devozerov <vo...@gridgain.com>
Authored: Fri Dec 9 16:05:24 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Dec 9 16:05:24 2016 +0300
----------------------------------------------------------------------
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 19 ---------------
...niteCachePartitionedFieldsQuerySelfTest.java | 25 ++++++++++++++++++++
2 files changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[03/50] [abbrv] ignite git commit: IGNITE-4350 Reworked
JdbcTypesDefaultTransformed logic. Improved improved error messages in
CacheJdbcPojoStore.
Posted by sb...@apache.org.
IGNITE-4350 Reworked JdbcTypesDefaultTransformed logic. Improved improved error messages in CacheJdbcPojoStore.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bc977d32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bc977d32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bc977d32
Branch: refs/heads/master
Commit: bc977d3211906ef94e1f7d3f0f988efbed65034f
Parents: 391f4be
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Fri Dec 9 16:11:31 2016 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Dec 9 16:11:31 2016 +0700
----------------------------------------------------------------------
.../cache/store/jdbc/CacheJdbcPojoStore.java | 13 +-
.../store/jdbc/JdbcTypesDefaultTransformer.java | 112 ++++----
.../jdbc/JdbcTypesDefaultTransformerTest.java | 283 +++++++++++++++++++
.../ignite/testsuites/IgniteCacheTestSuite.java | 2 +
4 files changed, 349 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bc977d32/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
index 209dd11..b9f6e8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStore.java
@@ -150,15 +150,15 @@ public class CacheJdbcPojoStore<K, V> extends CacheAbstractJdbcStore<K, V> {
*/
private Object buildBuiltinObject(String typeName, JdbcTypeField[] fields, Map<String, Integer> loadColIdxs,
ResultSet rs) throws CacheLoaderException {
- try {
- JdbcTypeField field = fields[0];
+ JdbcTypeField field = fields[0];
+ try {
Integer colIdx = columnIndex(loadColIdxs, field.getDatabaseFieldName());
return transformer.getColumnValue(rs, colIdx, field.getJavaFieldType());
}
catch (SQLException e) {
- throw new CacheLoaderException("Failed to read object of class: " + typeName, e);
+ throw new CacheLoaderException("Failed to read object: [cls=" + typeName + ", prop=" + field + "]", e);
}
}
@@ -211,12 +211,13 @@ public class CacheJdbcPojoStore<K, V> extends CacheAbstractJdbcStore<K, V> {
}
catch (Exception e) {
throw new CacheLoaderException("Failed to set property in POJO class [type=" + typeName +
- ", prop=" + fldJavaName + ", col=" + colIdx + ", dbName=" + dbName + "]", e);
+ ", colIdx=" + colIdx + ", prop=" + fld +
+ ", dbValCls=" + colVal.getClass().getName() + ", dbVal=" + colVal + "]", e);
}
}
catch (SQLException e) {
- throw new CacheLoaderException("Failed to read object property [type= " + typeName +
- ", prop=" + fldJavaName + ", col=" + colIdx + ", dbName=" + dbName + "]", e);
+ throw new CacheLoaderException("Failed to read object property [type=" + typeName +
+ ", colIdx=" + colIdx + ", prop=" + fld + "]", e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/bc977d32/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
index e91c2d3..c32eaa2 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformer.java
@@ -17,9 +17,12 @@
package org.apache.ignite.cache.store.jdbc;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Time;
import java.sql.Timestamp;
import java.util.UUID;
@@ -35,56 +38,71 @@ public class JdbcTypesDefaultTransformer implements JdbcTypesTransformer {
/** {@inheritDoc} */
@Override public Object getColumnValue(ResultSet rs, int colIdx, Class<?> type) throws SQLException {
- Object val = rs.getObject(colIdx);
+ if (type == String.class)
+ return rs.getString(colIdx);
- if (val == null)
- return null;
+ if (type == int.class || type == Integer.class) {
+ int res = rs.getInt(colIdx);
- if (type == int.class)
- return rs.getInt(colIdx);
+ return rs.wasNull() && type == Integer.class ? null : res;
+ }
+
+ if (type == long.class || type == Long.class) {
+ long res = rs.getLong(colIdx);
+
+ return rs.wasNull() && type == Long.class ? null : res;
+ }
+
+ if (type == double.class || type == Double.class) {
+ double res = rs.getDouble(colIdx);
+
+ return rs.wasNull() && type == Double.class ? null : res;
+ }
+
+ if (type == Date.class || type == java.util.Date.class)
+ return rs.getDate(colIdx);
+
+ if (type == Timestamp.class)
+ return rs.getTimestamp(colIdx);
+
+ if (type == Time.class)
+ return rs.getTime(colIdx);
- if (type == long.class)
- return rs.getLong(colIdx);
+ if (type == boolean.class || type == Boolean.class) {
+ boolean res = rs.getBoolean(colIdx);
- if (type == double.class)
- return rs.getDouble(colIdx);
+ return rs.wasNull() && type == Boolean.class ? null : res;
+ }
- if (type == boolean.class || type == Boolean.class)
- return rs.getBoolean(colIdx);
+ if (type == byte.class || type == Byte.class) {
+ byte res = rs.getByte(colIdx);
- if (type == byte.class)
- return rs.getByte(colIdx);
+ return rs.wasNull() && type == Byte.class ? null : res;
+ }
- if (type == short.class)
- return rs.getShort(colIdx);
+ if (type == short.class || type == Short.class) {
+ short res = rs.getShort(colIdx);
- if (type == float.class)
- return rs.getFloat(colIdx);
+ return rs.wasNull() && type == Short.class ? null : res;
+ }
- if (type == Integer.class || type == Long.class || type == Double.class ||
- type == Byte.class || type == Short.class || type == Float.class) {
- Number num = (Number)val;
+ if (type == float.class || type == Float.class) {
+ float res = rs.getFloat(colIdx);
- if (type == Integer.class)
- return num.intValue();
- else if (type == Long.class)
- return num.longValue();
- else if (type == Double.class)
- return num.doubleValue();
- else if (type == Byte.class)
- return num.byteValue();
- else if (type == Short.class)
- return num.shortValue();
- else if (type == Float.class)
- return num.floatValue();
+ return rs.wasNull() && type == Float.class ? null : res;
}
+ if (type == BigDecimal.class)
+ return rs.getBigDecimal(colIdx);
+
if (type == UUID.class) {
- if (val instanceof UUID)
- return val;
+ Object res = rs.getObject(colIdx);
+
+ if (res instanceof UUID)
+ return res;
- if (val instanceof byte[]) {
- ByteBuffer bb = ByteBuffer.wrap((byte[])val);
+ if (res instanceof byte[]) {
+ ByteBuffer bb = ByteBuffer.wrap((byte[])res);
long most = bb.getLong();
long least = bb.getLong();
@@ -92,26 +110,10 @@ public class JdbcTypesDefaultTransformer implements JdbcTypesTransformer {
return new UUID(most, least);
}
- if (val instanceof String)
- return UUID.fromString((String)val);
- }
-
- // Workaround for known issue with Oracle JDBC driver https://community.oracle.com/thread/2355464?tstart=0
- if (type == java.sql.Date.class && val instanceof java.util.Date)
- return new java.sql.Date(((java.util.Date)val).getTime());
-
- // Workaround for known issue with Oracle JDBC driver and timestamp.
- // http://stackoverflow.com/questions/13269564/java-lang-classcastexception-oracle-sql-timestamp-cannot-be-cast-to-java-sql-ti
- if (type == Timestamp.class && !(val instanceof Timestamp) &&
- val.getClass().getName().startsWith("oracle.sql.TIMESTAMP")) {
- try {
- return val.getClass().getMethod("timestampValue").invoke(val);
- }
- catch (Exception e) {
- throw new SQLException("Failed to read data of oracle.sql.TIMESTAMP type.", e);
- }
+ if (res instanceof String)
+ return UUID.fromString((String)res);
}
- return val;
+ return rs.getObject(colIdx);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/bc977d32/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java
new file mode 100644
index 0000000..5e490f7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/store/jdbc/JdbcTypesDefaultTransformerTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache.store.jdbc;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.UUID;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for {@link JdbcTypesDefaultTransformer}.
+ */
+public class JdbcTypesDefaultTransformerTest extends GridCommonAbstractTest {
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTransformer() throws Exception {
+ // Connection to H2.
+ String jdbcUrl = "jdbc:h2:mem:JdbcTypesDefaultTransformerTest";
+ String usr = "sa";
+ String pwd = "";
+
+ // Connection to Oracle.
+ // -Duser.region=us -Duser.language=en
+// Class.forName("oracle.jdbc.OracleDriver");
+// String jdbcUrl = "jdbc:oracle:thin:@localhost:1521:XE";
+// String usr = "test";
+// String pwd = "test";
+
+ // Connection to MS SQL.
+// Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+// String jdbcUrl = "jdbc:sqlserver://localhost;databaseName=master";
+// String usr = "test";
+// String pwd = "test";
+
+ // Connection to DB2.
+// Class.forName("com.ibm.db2.jcc.DB2Driver");
+// String jdbcUrl = "jdbc:db2://localhost:50000/sample";
+// String usr = "test";
+// String pwd = "test";
+
+ // Connection to Postgre SQL.
+// Class.forName("org.postgresql.Driver");
+// String jdbcUrl = "jdbc:postgresql://localhost:5433/postgres";
+// String usr = "test";
+// String pwd = "test";
+
+ // Connection to My SQL.
+// Class.forName("com.mysql.jdbc.Driver");
+// String jdbcUrl = "jdbc:mysql://localhost:3306/test";
+// String usr = "test";
+// String pwd = "test";
+
+ try (Connection conn = DriverManager.getConnection(jdbcUrl, usr, pwd)) {
+ Statement stmt = conn.createStatement();
+
+ try {
+ stmt.executeUpdate("DROP TABLE TEST_TRANSFORMER");
+ }
+ catch (SQLException ignored) {
+ // No-op.
+ }
+
+ // Create table in H2.
+ stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+ "c1 BOOLEAN, c2 INTEGER, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+ "c9 TIME, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 UUID)");
+
+ // Create table in ORACLE.
+// stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+// "c1 NUMBER(1), c2 INTEGER, c3 NUMBER(3), c4 NUMBER(4), c5 NUMBER(20), c6 NUMBER(20, 2), c7 NUMBER(20, 2), c8 NUMBER(10, 2), " +
+// "c9 TIMESTAMP, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 VARCHAR(36))");
+
+ // Create table in MS SQL.
+// stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+// "c1 BIT, c2 INTEGER, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+// "c9 TIME, c10 DATE, c11 DATETIME, c12 VARCHAR(100), c13 VARCHAR(36))");
+
+ // Create table in DB2.
+// stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+// "c1 SMALLINT , c2 INTEGER, c3 SMALLINT , c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+// "c9 TIME, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 VARCHAR(36))");
+
+ // Create table in Postgre SQL.
+// stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+// "c1 BOOLEAN, c2 INTEGER, c3 SMALLINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+// "c9 TIME, c10 DATE, c11 TIMESTAMP, c12 VARCHAR(100), c13 UUID)");
+
+ // Create table in MySQL.
+// stmt.executeUpdate("CREATE TABLE TEST_TRANSFORMER(id INTEGER, " +
+// "c1 BOOLEAN, c2 INTEGER, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 DECIMAL(20, 2), c7 DOUBLE PRECISION, c8 REAL, " +
+// "c9 TIME, c10 DATE, c11 TIMESTAMP(3), c12 VARCHAR(100), c13 VARCHAR(36))");
+
+ // Add data to H2, Postgre SQL and MySQL.
+ stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+ "VALUES (1, true, 1, 2, 3, 4, 5.35, 6.15, 7.32, '00:01:08', '2016-01-01', '2016-01-01 00:01:08.296', " +
+ "'100', '736bc956-090c-40d2-94da-916f2161f8a2')");
+ stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+ "VALUES (2, false, 10, 20, 30, 40, 50, 60, 70, current_time, current_date, current_timestamp, " +
+ "'100.55', '736bc956-090c-40d2-94da-916f2161cdea')");
+
+ // Add data to Oracle.
+// stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+// "VALUES (1, 1, 1, 2, 3, 4, 5.35, 6.15, 7.32, " +
+// "TO_TIMESTAMP('2016-01-01 00:01:08', 'YYYY-MM-DD HH24:MI:SS'), " +
+// "TO_DATE('2016-01-01', 'YYYY-MM-DD')," +
+// "TO_TIMESTAMP('2016-01-01 00:01:08.296', 'YYYY-MM-DD HH24:MI:SS.FF3'), " +
+// "'100', '736bc956-090c-40d2-94da-916f2161f8a2')");
+// stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+// "VALUES (2, 0, 10, 20, 30, 40, 50, 60, 70," +
+// "TO_TIMESTAMP('2016-01-01 00:01:08', 'YYYY-MM-DD HH24:MI:SS'), " +
+// "TO_DATE('2016-01-01', 'YYYY-MM-DD')," +
+// "TO_TIMESTAMP('2016-01-01 00:01:08.296', 'YYYY-MM-DD HH24:MI:SS.FF3'), " +
+// "'100.55', '736bc956-090c-40d2-94da-916f2161cdea')");
+
+ // Add data to MS SQL or IBM DB2.
+// stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+// "VALUES (1, 1, 1, 2, 3, 4, 5.35, 6.15, 7.32, '00:01:08', '2016-01-01', '2016-01-01 00:01:08.296', " +
+// "'100', '736bc956-090c-40d2-94da-916f2161f8a2')");
+// stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+// "VALUES (2, 0, 10, 20, 30, 40, 50, 60, 70, '00:01:08', '2016-01-01', '2016-01-01 00:01:08.296', " +
+// "'100.55', '736bc956-090c-40d2-94da-916f2161cdea')");
+
+ stmt.executeUpdate("INSERT INTO TEST_TRANSFORMER(id, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) " +
+ "VALUES (3, null, null, null, null, null, null, null, null, null, null, null, null, null)");
+
+ ResultSet rs = stmt.executeQuery("select c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13 from TEST_TRANSFORMER order by id");
+
+ assertTrue(rs.next());
+
+ JdbcTypesDefaultTransformer transformer = JdbcTypesDefaultTransformer.INSTANCE;
+
+ // c1: Test BOOLEAN column.
+ assertTrue((Boolean)transformer.getColumnValue(rs, 1, boolean.class));
+ assertTrue((Boolean)transformer.getColumnValue(rs, 1, Boolean.class));
+ assertEquals(1, transformer.getColumnValue(rs, 1, int.class));
+ assertEquals(1.0, transformer.getColumnValue(rs, 1, double.class));
+
+ String s = (String)transformer.getColumnValue(rs, 1, String.class);
+ assertTrue("true".equalsIgnoreCase(s) || "1".equals(s) || "t".equalsIgnoreCase(s));
+
+ // c2: Test INTEGER column.
+ assertEquals(1, transformer.getColumnValue(rs, 2, int.class));
+ assertEquals(1, transformer.getColumnValue(rs, 2, Integer.class));
+ assertEquals(1L, transformer.getColumnValue(rs, 2, Long.class));
+ assertEquals(1.0, transformer.getColumnValue(rs, 2, double.class));
+ assertEquals("1", transformer.getColumnValue(rs, 2, String.class));
+
+ // c3: Test TINYINT column.
+ byte b = 2;
+ assertEquals(b, transformer.getColumnValue(rs, 3, byte.class));
+ assertEquals(b, transformer.getColumnValue(rs, 3, Byte.class));
+ assertEquals(2.0, transformer.getColumnValue(rs, 3, double.class));
+ assertEquals("2", transformer.getColumnValue(rs, 3, String.class));
+
+ // c4: Test SMALLINT column.
+ short shrt = 3;
+ assertEquals(shrt, transformer.getColumnValue(rs, 4, short.class));
+ assertEquals(shrt, transformer.getColumnValue(rs, 4, Short.class));
+ assertEquals(3.0, transformer.getColumnValue(rs, 4, double.class));
+ assertEquals("3", transformer.getColumnValue(rs, 4, String.class));
+
+ // c5: Test BIGINT column.
+ assertEquals(4L, transformer.getColumnValue(rs, 5, long.class));
+ assertEquals(4L, transformer.getColumnValue(rs, 5, Long.class));
+ assertEquals(4, transformer.getColumnValue(rs, 5, int.class));
+ assertEquals(4, transformer.getColumnValue(rs, 5, Integer.class));
+ assertEquals(4.0, transformer.getColumnValue(rs, 5, double.class));
+ assertEquals("4", transformer.getColumnValue(rs, 5, String.class));
+ assertEquals(new BigDecimal("4"), transformer.getColumnValue(rs, 5, BigDecimal.class));
+
+ // c6: Test DECIMAL column.
+ assertEquals(new BigDecimal("5.35"), transformer.getColumnValue(rs, 6, BigDecimal.class));
+ assertEquals(5L, transformer.getColumnValue(rs, 6, long.class));
+ assertEquals("5.35", transformer.getColumnValue(rs, 6, String.class));
+
+ // c7: Test DOUBLE column.
+ assertEquals(6.15, transformer.getColumnValue(rs, 7, double.class));
+ assertEquals(6.15, transformer.getColumnValue(rs, 7, Double.class));
+ assertEquals(6, transformer.getColumnValue(rs, 7, int.class));
+ assertEquals(6, transformer.getColumnValue(rs, 7, Integer.class));
+ assertTrue(transformer.getColumnValue(rs, 7, String.class).toString().startsWith("6.15"));
+
+ // c8: Test REAL column.
+ assertTrue((7.32f - (Float)transformer.getColumnValue(rs, 8, float.class)) < 0.01);
+ assertTrue((7.32f - (Float)transformer.getColumnValue(rs, 8, Float.class)) < 0.01);
+ assertTrue((7.32 - (Double)transformer.getColumnValue(rs, 8, double.class)) < 0.01);
+ assertTrue((7.32 - (Double)transformer.getColumnValue(rs, 8, Double.class)) < 0.01);
+ assertEquals(7, transformer.getColumnValue(rs, 8, int.class));
+ assertEquals(7, transformer.getColumnValue(rs, 8, Integer.class));
+ assertTrue(transformer.getColumnValue(rs, 8, String.class).toString().startsWith("7.32"));
+
+ // c9: Test TIME column.
+ assertTrue(transformer.getColumnValue(rs, 9, Time.class) instanceof Time);
+ assertTrue(transformer.getColumnValue(rs, 9, String.class).toString().contains("00:01:08"));
+
+ // c10: Test DATE column.
+ assertTrue(transformer.getColumnValue(rs, 10, Date.class) instanceof Date);
+ assertTrue(transformer.getColumnValue(rs, 10, String.class).toString().startsWith("2016-01-01"));
+
+ // c11: Test TIMESTAMP column.
+ transformer.getColumnValue(rs, 11, Timestamp.class);
+ assertTrue(transformer.getColumnValue(rs, 11, String.class).toString().startsWith("2016-01-01 00:01:08.29"));
+
+ // c12: Test VARCHAR column.
+ assertEquals("100", transformer.getColumnValue(rs, 12, String.class));
+ assertEquals(100, transformer.getColumnValue(rs, 12, int.class));
+
+ // c13: Test UUID column.
+ transformer.getColumnValue(rs, 13, UUID.class);
+ assertEquals("736bc956-090c-40d2-94da-916f2161f8a2", transformer.getColumnValue(rs, 13, String.class));
+
+ assertTrue(rs.next());
+
+ // Test BOOLEAN column.
+ assertFalse((Boolean)transformer.getColumnValue(rs, 1, boolean.class));
+ assertFalse((Boolean)transformer.getColumnValue(rs, 1, Boolean.class));
+ assertEquals(0, transformer.getColumnValue(rs, 1, int.class));
+ assertEquals(0.0, transformer.getColumnValue(rs, 1, double.class));
+
+ s = (String)transformer.getColumnValue(rs, 1, String.class);
+ assertTrue("false".equalsIgnoreCase(s) || "0".equals(s) || "f".equalsIgnoreCase(s));
+
+ assertTrue(rs.next());
+
+ // Check how null values will be transformed.
+ assertNotNull(transformer.getColumnValue(rs, 1, boolean.class));
+ assertNull(transformer.getColumnValue(rs, 1, Boolean.class));
+
+ assertEquals(0, transformer.getColumnValue(rs, 2, int.class));
+ assertNull(transformer.getColumnValue(rs, 2, Integer.class));
+
+ assertEquals((byte)0, transformer.getColumnValue(rs, 3, byte.class));
+ assertNull(transformer.getColumnValue(rs, 3, Byte.class));
+
+ assertEquals((short)0, transformer.getColumnValue(rs, 4, short.class));
+ assertNull(transformer.getColumnValue(rs, 4, Short.class));
+
+ assertEquals(0L, transformer.getColumnValue(rs, 5, long.class));
+ assertNull(transformer.getColumnValue(rs, 5, Long.class));
+
+ assertNull(transformer.getColumnValue(rs, 6, BigDecimal.class));
+
+ assertEquals(0d, transformer.getColumnValue(rs, 7, double.class));
+ assertNull(transformer.getColumnValue(rs, 7, Double.class));
+
+ assertEquals(0f, transformer.getColumnValue(rs, 8, float.class));
+ assertNull(transformer.getColumnValue(rs, 8, Float.class));
+
+ assertNull(transformer.getColumnValue(rs, 9, Time.class));
+
+ assertNull(transformer.getColumnValue(rs, 10, Date.class));
+
+ assertNull(transformer.getColumnValue(rs, 11, Timestamp.class));
+
+ assertNull(transformer.getColumnValue(rs, 12, String.class));
+
+ assertNull(transformer.getColumnValue(rs, 13, UUID.class));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/bc977d32/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index fd13e98..deec72a 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -39,6 +39,7 @@ import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreOptimizedMarshallerW
import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreTest;
import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreMultithreadedSelfTest;
import org.apache.ignite.cache.store.jdbc.GridCacheJdbcBlobStoreSelfTest;
+import org.apache.ignite.cache.store.jdbc.JdbcTypesDefaultTransformerTest;
import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
import org.apache.ignite.internal.processors.cache.CacheEntryProcessorCopySelfTest;
@@ -246,6 +247,7 @@ public class IgniteCacheTestSuite extends TestSuite {
suite.addTestSuite(GridCacheConfigurationConsistencySelfTest.class);
suite.addTestSuite(GridCacheJdbcBlobStoreSelfTest.class);
suite.addTestSuite(GridCacheJdbcBlobStoreMultithreadedSelfTest.class);
+ suite.addTestSuite(JdbcTypesDefaultTransformerTest.class);
suite.addTestSuite(CacheJdbcPojoStoreTest.class);
suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerSelfTest.class);
suite.addTestSuite(CacheJdbcPojoStoreOptimizedMarshallerWithSqlEscapeSelfTest.class);
[37/50] [abbrv] ignite git commit: IGNITE-3220 I/O bottleneck on
server/client cluster configuration Communications optimizations: -
possibility to open separate in/out connections - possibility to have
multiple connections between nodes - implemented NI
Posted by sb...@apache.org.
IGNITE-3220 I/O bottleneck on server/client cluster configuration
Communications optimizations:
- possibility to open separate in/out connections
- possibility to have multiple connections between nodes
- implemented NIO sessions balancing between NIO threads
- reduced amount of work and blocking calls in NIO threads
Other:
- implemented StripedExecutor for cache messages handling
- added 'io test' messages for IO performance testing
(cherry picked from commit 10ade28)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05dd08b9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05dd08b9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05dd08b9
Branch: refs/heads/master
Commit: 05dd08b993e2d7f88176c051463b178431714f85
Parents: 57eb47f
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 9 12:28:47 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 16 14:01:53 2016 +0300
----------------------------------------------------------------------
.../ignite/examples/ExampleNodeStartup.java | 2 +-
.../examples/datagrid/CachePutGetExample.java | 2 +-
.../scalar/examples/ScalarJvmCloudExample.scala | 2 +-
.../rest/ClientMemcachedProtocolSelfTest.java | 4 +-
.../rest/protocols/tcp/MockNioSession.java | 25 +-
.../apache/ignite/IgniteSystemProperties.java | 3 +
.../cache/store/CacheLoadOnlyStoreAdapter.java | 6 +-
.../configuration/IgniteConfiguration.java | 50 +-
.../internal/GridEventConsumeHandler.java | 2 +-
.../ignite/internal/GridJobContextImpl.java | 4 +-
.../ignite/internal/GridKernalContext.java | 9 +
.../ignite/internal/GridKernalContextImpl.java | 16 +-
.../internal/GridPerformanceSuggestions.java | 2 +-
.../org/apache/ignite/internal/GridTopic.java | 5 +-
.../ignite/internal/IgniteInternalFuture.java | 11 +
.../apache/ignite/internal/IgniteKernal.java | 85 +-
.../org/apache/ignite/internal/IgnitionEx.java | 32 +-
.../GridClientConnectionManagerAdapter.java | 1 +
.../client/router/impl/GridTcpRouterImpl.java | 1 +
.../managers/communication/GridIoManager.java | 207 ++-
.../managers/communication/GridIoMessage.java | 13 +
.../communication/GridIoMessageFactory.java | 12 +-
.../communication/IgniteIoTestMessage.java | 235 +++
.../processors/cache/GridCacheAdapter.java | 26 +-
.../processors/cache/GridCacheMessage.java | 7 +
.../processors/cache/GridCacheUtils.java | 35 +
.../processors/cache/IgniteCacheProxy.java | 8 +
.../distributed/GridDistributedLockRequest.java | 5 +
.../GridDistributedTxFinishResponse.java | 6 +
.../GridDistributedUnlockRequest.java | 5 +
.../distributed/dht/GridDhtCacheAdapter.java | 3 +-
.../distributed/dht/GridDhtLockResponse.java | 9 +-
.../dht/atomic/GridDhtAtomicCache.java | 5 +-
.../GridDhtAtomicSingleUpdateRequest.java | 5 +
.../dht/atomic/GridDhtAtomicUpdateRequest.java | 5 +
.../atomic/GridNearAtomicFullUpdateRequest.java | 5 +
.../GridNearAtomicSingleUpdateRequest.java | 5 +
.../distributed/near/GridNearGetRequest.java | 5 +
.../local/atomic/GridLocalAtomicCache.java | 3 +
.../query/GridCacheDistributedQueryManager.java | 2 +-
.../cache/query/GridCacheQueryRequest.java | 6 +-
.../transactions/IgniteTxLocalAdapter.java | 8 +-
.../datastreamer/DataStreamProcessor.java | 22 +-
.../internal/processors/igfs/IgfsContext.java | 4 +-
.../processors/igfs/IgfsDataManager.java | 6 +-
.../internal/processors/igfs/IgfsImpl.java | 2 +-
.../internal/processors/odbc/OdbcProcessor.java | 1 +
.../platform/compute/PlatformCompute.java | 6 +
.../tcp/GridTcpMemcachedNioListener.java | 15 +-
.../protocols/tcp/GridTcpRestNioListener.java | 2 +-
.../rest/protocols/tcp/GridTcpRestProtocol.java | 1 +
.../service/GridServiceProcessor.java | 6 +-
.../ignite/internal/util/IgniteUtils.java | 62 +-
.../ignite/internal/util/StripedExecutor.java | 667 +++++++++
.../util/future/GridFinishedFuture.java | 24 +
.../internal/util/future/GridFutureAdapter.java | 15 +-
.../util/future/GridFutureChainListener.java | 30 +-
.../internal/util/ipc/IpcToNioAdapter.java | 2 +-
.../nio/GridAbstractCommunicationClient.java | 12 +-
.../util/nio/GridCommunicationClient.java | 9 +-
.../nio/GridConnectionBytesVerifyFilter.java | 15 +-
.../util/nio/GridNioAsyncNotifyFilter.java | 10 +-
.../internal/util/nio/GridNioCodecFilter.java | 17 +-
.../ignite/internal/util/nio/GridNioFilter.java | 16 +-
.../internal/util/nio/GridNioFilterAdapter.java | 10 +-
.../internal/util/nio/GridNioFilterChain.java | 14 +-
.../ignite/internal/util/nio/GridNioFuture.java | 4 +-
.../util/nio/GridNioRecoveryDescriptor.java | 124 +-
.../ignite/internal/util/nio/GridNioServer.java | 1404 +++++++++++++++---
.../internal/util/nio/GridNioSession.java | 25 +-
.../internal/util/nio/GridNioSessionImpl.java | 65 +-
.../ignite/internal/util/nio/GridNioWorker.java | 48 +
.../util/nio/GridSelectorNioSessionImpl.java | 221 ++-
.../util/nio/GridShmemCommunicationClient.java | 7 +-
.../util/nio/GridTcpNioCommunicationClient.java | 55 +-
.../internal/util/nio/SessionWriteRequest.java | 85 ++
.../internal/util/nio/ssl/GridNioSslFilter.java | 10 +-
.../util/nio/ssl/GridNioSslHandler.java | 4 +-
.../util/tostring/GridToStringBuilder.java | 2 +-
.../communication/tcp/TcpCommunicationSpi.java | 1340 ++++++++++++-----
.../tcp/TcpCommunicationSpiMBean.java | 40 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 14 +-
.../ignite/stream/socket/SocketStreamer.java | 1 +
.../ignite/thread/IgniteThreadFactory.java | 8 +-
.../IgniteSlowClientDetectionSelfTest.java | 1 +
...unicationBalanceMultipleConnectionsTest.java | 28 +
.../IgniteCommunicationBalanceTest.java | 339 +++++
.../communication/IgniteIoTestMessagesTest.java | 95 ++
.../IgniteVariousConnectionNumberTest.java | 166 +++
.../cache/CrossCacheTxRandomOperationsTest.java | 30 +-
...idAbstractCacheInterceptorRebalanceTest.java | 4 +-
...CacheOffHeapMultiThreadedUpdateSelfTest.java | 6 +-
...eAtomicMessageRecovery10ConnectionsTest.java | 28 +
...cMessageRecoveryNoPairedConnectionsTest.java | 47 +
...acheConnectionRecovery10ConnectionsTest.java | 35 +
.../distributed/IgniteCacheCreatePutTest.java | 2 +-
.../IgniteCacheMessageRecoveryAbstractTest.java | 24 +-
.../IgniteCacheMessageWriteTimeoutTest.java | 17 +-
.../dht/IgniteCacheMultiTxLockSelfTest.java | 6 +-
...erNoStripedPoolMultiNodeFullApiSelfTest.java | 35 +
...edNoStripedPoolMultiNodeFullApiSelfTest.java | 35 +
.../TxDeadlockDetectionNoHangsTest.java | 2 +-
.../TxOptimisticDeadlockDetectionTest.java | 29 +-
.../GridServiceProcessorProxySelfTest.java | 2 +-
.../util/future/GridFutureAdapterSelfTest.java | 122 +-
.../nio/impl/GridNioFilterChainSelfTest.java | 32 +-
.../loadtests/nio/GridNioBenchmarkClient.java | 4 +-
.../p2p/GridP2PRecursionTaskSelfTest.java | 2 +-
.../spi/GridTcpSpiForwardingSelfTest.java | 18 +-
.../GridTcpCommunicationSpiAbstractTest.java | 28 +-
...mmunicationSpiConcurrentConnectSelfTest.java | 82 +-
.../GridTcpCommunicationSpiConfigSelfTest.java | 5 +-
...cpCommunicationSpiMultithreadedSelfTest.java | 23 +-
...dTcpCommunicationSpiRecoveryAckSelfTest.java | 9 +-
...tionSpiRecoveryFailureDetectionSelfTest.java | 1 +
...ationSpiRecoveryNoPairedConnectionsTest.java | 28 +
...GridTcpCommunicationSpiRecoverySelfTest.java | 67 +-
...CommunicationRecoveryAckClosureSelfTest.java | 9 +-
.../junits/GridTestKernalContext.java | 4 +-
.../IgniteCacheFullApiSelfTestSuite.java | 6 +
.../ignite/testsuites/IgniteCacheTestSuite.java | 17 +-
.../IgniteSpiCommunicationSelfTestSuite.java | 2 +
.../hadoop/jobtracker/HadoopJobTracker.java | 4 +-
.../HadoopExternalCommunication.java | 5 +-
.../communication/HadoopIpcToNioAdapter.java | 2 +-
.../communication/HadoopMarshallerFilter.java | 6 +-
.../ignite/stream/kafka/KafkaStreamer.java | 2 +-
.../ignite/tools/classgen/ClassesGenerator.java | 8 +-
.../ignite/yardstick/IgniteBenchmarkUtils.java | 6 +-
.../yardstick/cache/CacheEntryEventProbe.java | 2 +-
.../yardstick/cache/IgniteIoTestBenchmark.java | 73 +
.../io/IgniteIoTestAbstractBenchmark.java | 61 +
.../io/IgniteIoTestSendAllBenchmark.java | 32 +
.../io/IgniteIoTestSendRandomBenchmark.java | 35 +
134 files changed, 5935 insertions(+), 998 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java b/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
index ad12297..dd8a72b 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ExampleNodeStartup.java
@@ -33,4 +33,4 @@ public class ExampleNodeStartup {
public static void main(String[] args) throws IgniteException {
Ignition.start("examples/config/example-ignite.xml");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
index 82a76b8..b9bae5b 100644
--- a/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/CachePutGetExample.java
@@ -105,4 +105,4 @@ public class CachePutGetExample {
for (Map.Entry<Integer, String> e : vals.entrySet())
System.out.println("Got entry [key=" + e.getKey() + ", val=" + e.getValue() + ']');
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala b/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
index 1014726..814bb2e 100644
--- a/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
+++ b/examples/src/main/scala/org/apache/ignite/scalar/examples/ScalarJvmCloudExample.scala
@@ -50,7 +50,7 @@ object ScalarJvmCloudExample {
val pool = Executors.newFixedThreadPool(NODES.size)
// Concurrently startup all nodes.
- NODES.foreach(name => pool.submit(new Runnable {
+ NODES.foreach(name => pool.execute(new Runnable {
@impl def run() {
// All defaults.
val cfg = new IgniteConfiguration
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
index 0f56c73..c03c06e 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/ClientMemcachedProtocolSelfTest.java
@@ -111,6 +111,8 @@ public class ClientMemcachedProtocolSelfTest extends AbstractRestProcessorSelfTe
Map<String, Object> map = client.getBulk("getKey1", "getKey2");
+ info("Map: " + map);
+
Assert.assertEquals(2, map.size());
Assert.assertEquals("getVal1", map.get("getKey1"));
@@ -443,4 +445,4 @@ public class ClientMemcachedProtocolSelfTest extends AbstractRestProcessorSelfTe
return res;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
index c82c73e..9bc4e7f 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/protocols/tcp/MockNioSession.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.rest.protocols.tcp;
import java.net.InetSocketAddress;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
import org.apache.ignite.internal.util.nio.GridNioFinishedFuture;
import org.apache.ignite.internal.util.nio.GridNioFuture;
@@ -111,6 +112,11 @@ public class MockNioSession extends GridMetadataAwareAdapter implements GridNioS
}
/** {@inheritDoc} */
+ @Override public void sendNoFuture(Object msg) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public GridNioFuture<Object> resumeReads() {
return null;
}
@@ -131,12 +137,27 @@ public class MockNioSession extends GridMetadataAwareAdapter implements GridNioS
}
/** {@inheritDoc} */
- @Override public void recoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ @Override public void outRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void inRecoveryDescriptor(GridNioRecoveryDescriptor recoveryDesc) {
// No-op.
}
/** {@inheritDoc} */
- @Nullable @Override public GridNioRecoveryDescriptor recoveryDescriptor() {
+ @Nullable @Override public GridNioRecoveryDescriptor outRecoveryDescriptor() {
return null;
}
+
+ /** {@inheritDoc} */
+ @Nullable @Override public GridNioRecoveryDescriptor inRecoveryDescriptor() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void systemMessage(Object msg) {
+ // No-op.
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index d0c0d5e..9650a31 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -477,6 +477,9 @@ public final class IgniteSystemProperties {
@Deprecated
public static final String IGNITE_BINARY_DONT_WRAP_TREE_STRUCTURES = "IGNITE_BINARY_DONT_WRAP_TREE_STRUCTURES";
+ /** */
+ public static final String IGNITE_IO_BALANCE_PERIOD = "IGNITE_IO_BALANCE_PERIOD";
+
/**
* When set to {@code true} fields are written by BinaryMarshaller in sorted order. Otherwise
* the natural order is used.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
index 7494e37..d3f381e 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/store/CacheLoadOnlyStoreAdapter.java
@@ -153,14 +153,14 @@ public abstract class CacheLoadOnlyStoreAdapter<K, V, I> implements CacheStore<K
buf.add(iter.next());
if (buf.size() == batchSize) {
- exec.submit(new Worker(c, buf, args));
+ exec.execute(new Worker(c, buf, args));
buf = new ArrayList<>(batchSize);
}
}
if (!buf.isEmpty())
- exec.submit(new Worker(c, buf, args));
+ exec.execute(new Worker(c, buf, args));
}
catch (RejectedExecutionException ignored) {
// Because of custom RejectedExecutionHandler.
@@ -330,4 +330,4 @@ public abstract class CacheLoadOnlyStoreAdapter<K, V, I> implements CacheStore<K
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index 75145a3..dcd8a80 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -146,7 +146,7 @@ public class IgniteConfiguration {
public static final int AVAILABLE_PROC_CNT = Runtime.getRuntime().availableProcessors();
/** Default core size of public thread pool. */
- public static final int DFLT_PUBLIC_THREAD_CNT = Math.max(8, AVAILABLE_PROC_CNT) * 2;
+ public static final int DFLT_PUBLIC_THREAD_CNT = Math.max(8, AVAILABLE_PROC_CNT);
/** Default keep alive time for public thread pool. */
@Deprecated
@@ -236,6 +236,12 @@ public class IgniteConfiguration {
/** Async Callback pool size. */
private int callbackPoolSize = DFLT_PUBLIC_THREAD_CNT;
+ /**
+ * Use striped pool for internal requests processing when possible
+ * (e.g. cache requests per-partition striping).
+ */
+ private int stripedPoolSize = DFLT_PUBLIC_THREAD_CNT;
+
/** System pool size. */
private int sysPoolSize = DFLT_SYSTEM_CORE_THREAD_CNT;
@@ -553,6 +559,7 @@ public class IgniteConfiguration {
sndRetryDelay = cfg.getNetworkSendRetryDelay();
sslCtxFactory = cfg.getSslContextFactory();
storeSesLsnrs = cfg.getCacheStoreSessionListenerFactories();
+ stripedPoolSize = cfg.getStripedPoolSize();
svcCfgs = cfg.getServiceConfiguration();
sysPoolSize = cfg.getSystemThreadPoolSize();
timeSrvPortBase = cfg.getTimeServerPortBase();
@@ -712,6 +719,47 @@ public class IgniteConfiguration {
}
/**
+ * Returns striped pool size that should be used for cache requests
+ * processing.
+ * <p>
+ * If set to non-positive value then requests get processed in system pool.
+ * <p>
+ * Striped pool is better for typical cache operations.
+ *
+ * @return Positive value if striped pool should be initialized
+ * with configured number of threads (stripes) and used for requests processing
+ * or non-positive value to process requests in system pool.
+ *
+ * @see #getPublicThreadPoolSize()
+ * @see #getSystemThreadPoolSize()
+ */
+ public int getStripedPoolSize() {
+ return stripedPoolSize;
+ }
+
+ /**
+ * Sets striped pool size that should be used for cache requests
+ * processing.
+ * <p>
+ * If set to non-positive value then requests get processed in system pool.
+ * <p>
+ * Striped pool is better for typical cache operations.
+ *
+ * @param stripedPoolSize Positive value if striped pool should be initialized
+ * with passed in number of threads (stripes) and used for requests processing
+ * or non-positive value to process requests in system pool.
+ * @return {@code this} for chaining.
+ *
+ * @see #getPublicThreadPoolSize()
+ * @see #getSystemThreadPoolSize()
+ */
+ public IgniteConfiguration setStripedPoolSize(int stripedPoolSize) {
+ this.stripedPoolSize = stripedPoolSize;
+
+ return this;
+ }
+
+ /**
* Should return a thread pool size to be used in grid.
* This executor service will be in charge of processing {@link ComputeJob GridJobs}
* and user messages sent to node.
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
index 715f8a5..68d34ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
@@ -181,7 +181,7 @@ class GridEventConsumeHandler implements GridContinuousHandler {
notificationQueue.add(new T3<>(nodeId, routineId, evt));
if (!notificationInProgress) {
- ctx.getSystemExecutorService().submit(new Runnable() {
+ ctx.getSystemExecutorService().execute(new Runnable() {
@Override public void run() {
if (!ctx.continuous().lockStopping())
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
index 804d228..dbfa0b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobContextImpl.java
@@ -217,7 +217,7 @@ public class GridJobContextImpl implements ComputeJobContext, Externalizable {
assert execSvc != null;
- execSvc.submit(new Runnable() {
+ execSvc.execute(new Runnable() {
@Override public void run() {
callcc0();
}
@@ -300,4 +300,4 @@ public class GridJobContextImpl implements ComputeJobContext, Externalizable {
@Override public String toString() {
return S.toString(GridJobContextImpl.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index ae29223..927944f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -66,6 +66,7 @@ import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.IgniteExceptionRegistry;
+import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.plugin.PluginNotFoundException;
import org.apache.ignite.plugin.PluginProvider;
@@ -511,6 +512,14 @@ public interface GridKernalContext extends Iterable<GridComponent> {
public ExecutorService getSystemExecutorService();
/**
+ * Executor service that is in charge of processing internal system messages
+ * in stripes (dedicated threads).
+ *
+ * @return Thread pool implementation to be used in grid for internal system messages.
+ */
+ public StripedExecutor getStripedExecutorService();
+
+ /**
* Executor service that is in charge of processing internal and Visor
* {@link org.apache.ignite.compute.ComputeJob GridJobs}.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 94c6448..a2ad1b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -83,6 +83,7 @@ import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.IgniteExceptionRegistry;
+import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.spring.IgniteSpringHelper;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -300,6 +301,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
/** */
@GridToStringExclude
+ protected StripedExecutor stripedExecSvc;
+
+ /** */
+ @GridToStringExclude
private ExecutorService p2pExecSvc;
/** */
@@ -381,6 +386,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
* @param marshCachePool Marshaller cache pool.
* @param execSvc Public executor service.
* @param sysExecSvc System executor service.
+ * @param stripedExecSvc Striped executor.
* @param p2pExecSvc P2P executor service.
* @param mgmtExecSvc Management executor service.
* @param igfsExecSvc IGFS executor service.
@@ -400,6 +406,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
ExecutorService marshCachePool,
ExecutorService execSvc,
ExecutorService sysExecSvc,
+ StripedExecutor stripedExecSvc,
ExecutorService p2pExecSvc,
ExecutorService mgmtExecSvc,
ExecutorService igfsExecSvc,
@@ -407,7 +414,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
ExecutorService affExecSvc,
@Nullable ExecutorService idxExecSvc,
IgniteStripedThreadPoolExecutor callbackExecSvc,
- List<PluginProvider> plugins) throws IgniteCheckedException {
+ List<PluginProvider> plugins
+ ) throws IgniteCheckedException {
assert grid != null;
assert cfg != null;
assert gw != null;
@@ -419,6 +427,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
this.marshCachePool = marshCachePool;
this.execSvc = execSvc;
this.sysExecSvc = sysExecSvc;
+ this.stripedExecSvc = stripedExecSvc;
this.p2pExecSvc = p2pExecSvc;
this.mgmtExecSvc = mgmtExecSvc;
this.igfsExecSvc = igfsExecSvc;
@@ -948,6 +957,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
}
/** {@inheritDoc} */
+ @Override public StripedExecutor getStripedExecutorService() {
+ return stripedExecSvc;
+ }
+
+ /** {@inheritDoc} */
@Override public ExecutorService getManagementExecutorService() {
return mgmtExecSvc;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java b/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
index b040a97..5e8e520 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridPerformanceSuggestions.java
@@ -89,4 +89,4 @@ public class GridPerformanceSuggestions {
@Override public String toString() {
return S.toString(GridPerformanceSuggestions.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index b5608db..24ddcd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -97,7 +97,10 @@ public enum GridTopic {
TOPIC_QUERY,
/** */
- TOPIC_TX;
+ TOPIC_TX,
+
+ /** */
+ TOPIC_IO_TEST;
/** Enum values. */
private static final GridTopic[] VALS = values();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
index b80a755..789556d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteInternalFuture.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal;
+import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.IgniteCheckedException;
@@ -133,6 +134,16 @@ public interface IgniteInternalFuture<R> {
public <T> IgniteInternalFuture<T> chain(IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb);
/**
+ * Make a chained future to convert result of this future (when complete) into a new format.
+ * It is guaranteed that done callback will be called only ONCE.
+ *
+ * @param doneCb Done callback that is applied to this future when it finishes to produce chained future result.
+ * @param exec Executor to run callback.
+ * @return Chained future that finishes after this future completes and done callback is called.
+ */
+ public <T> IgniteInternalFuture<T> chain(IgniteClosure<? super IgniteInternalFuture<R>, T> doneCb, Executor exec);
+
+ /**
* @return Error value if future has already been completed with error.
*/
public Throwable error();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 57aab00..7935e06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -61,10 +61,10 @@ import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteEvents;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteFileSystem;
+import org.apache.ignite.IgniteLock;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteMessaging;
import org.apache.ignite.IgniteQueue;
-import org.apache.ignite.IgniteLock;
import org.apache.ignite.IgniteScheduler;
import org.apache.ignite.IgniteSemaphore;
import org.apache.ignite.IgniteServices;
@@ -115,7 +115,6 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.hadoop.Hadoop;
-import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.job.GridJobProcessor;
import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
@@ -139,6 +138,7 @@ import org.apache.ignite.internal.processors.service.GridServiceProcessor;
import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
+import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -175,6 +175,7 @@ import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONFIG_URL;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DAEMON;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_NO_ASCII;
@@ -182,7 +183,6 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALL
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
import static org.apache.ignite.IgniteSystemProperties.getBoolean;
import static org.apache.ignite.IgniteSystemProperties.snapshot;
import static org.apache.ignite.internal.GridKernalState.DISCONNECTED;
@@ -199,7 +199,6 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_BUILD_VER;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CLIENT_MODE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_GRID_NAME;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
@@ -208,11 +207,12 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JMX_PORT;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_ARGS;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_PID;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LANG_RUNTIME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
@@ -663,6 +663,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* @param utilityCachePool Utility cache pool.
* @param execSvc Executor service.
* @param sysExecSvc System executor service.
+ * @param stripedExecSvc Striped executor.
* @param p2pExecSvc P2P executor service.
* @param mgmtExecSvc Management executor service.
* @param igfsExecSvc IGFS executor service.
@@ -673,11 +674,13 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* @throws IgniteCheckedException Thrown in case of any errors.
*/
@SuppressWarnings({"CatchGenericClass", "unchecked"})
- public void start(final IgniteConfiguration cfg,
+ public void start(
+ final IgniteConfiguration cfg,
ExecutorService utilityCachePool,
ExecutorService marshCachePool,
final ExecutorService execSvc,
final ExecutorService sysExecSvc,
+ final StripedExecutor stripedExecSvc,
ExecutorService p2pExecSvc,
ExecutorService mgmtExecSvc,
ExecutorService igfsExecSvc,
@@ -685,7 +688,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
ExecutorService affExecSvc,
@Nullable ExecutorService idxExecSvc,
IgniteStripedThreadPoolExecutor callbackExecSvc,
- GridAbsClosure errHnd)
+ GridAbsClosure errHnd
+ )
throws IgniteCheckedException
{
gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getGridName()));
@@ -785,6 +789,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
marshCachePool,
execSvc,
sysExecSvc,
+ stripedExecSvc,
p2pExecSvc,
mgmtExecSvc,
igfsExecSvc,
@@ -792,7 +797,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
affExecSvc,
idxExecSvc,
callbackExecSvc,
- plugins);
+ plugins
+ );
cfg.getMarshaller().setContext(ctx.marshallerContext());
@@ -986,24 +992,51 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
starveTask = ctx.timeout().schedule(new Runnable() {
/** Last completed task count. */
- private long lastCompletedCnt;
+ private long lastCompletedCntPub;
+
+ /** Last completed task count. */
+ private long lastCompletedCntSys;
@Override public void run() {
- if (!(execSvc instanceof ThreadPoolExecutor))
- return;
+ if (execSvc instanceof ThreadPoolExecutor) {
+ ThreadPoolExecutor exec = (ThreadPoolExecutor)execSvc;
+
+ lastCompletedCntPub = checkPoolStarvation(exec, lastCompletedCntPub, "public");
+ }
- ThreadPoolExecutor exec = (ThreadPoolExecutor)execSvc;
+ if (sysExecSvc instanceof ThreadPoolExecutor) {
+ ThreadPoolExecutor exec = (ThreadPoolExecutor)sysExecSvc;
+ lastCompletedCntSys = checkPoolStarvation(exec, lastCompletedCntSys, "system");
+ }
+
+ if (stripedExecSvc != null)
+ stripedExecSvc.checkStarvation();
+ }
+
+ /**
+ * @param exec Thread pool executor to check.
+ * @param lastCompletedCnt Last completed tasks count.
+ * @param pool Pool name for message.
+ * @return Current completed tasks count.
+ */
+ private long checkPoolStarvation(
+ ThreadPoolExecutor exec,
+ long lastCompletedCnt,
+ String pool
+ ) {
long completedCnt = exec.getCompletedTaskCount();
// If all threads are active and no task has completed since last time and there is
// at least one waiting request, then it is possible starvation.
if (exec.getPoolSize() == exec.getActiveCount() && completedCnt == lastCompletedCnt &&
!exec.getQueue().isEmpty())
- LT.warn(log, "Possible thread pool starvation detected (no task completed in last " +
- interval + "ms, is executorService pool size large enough?)");
+ LT.warn(
+ log,
+ "Possible thread pool starvation detected (no task completed in last " +
+ interval + "ms, is " + pool + " thread pool size large enough?)");
- lastCompletedCnt = completedCnt;
+ return completedCnt;
}
}, interval, interval);
}
@@ -1128,6 +1161,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}, longOpDumpTimeout, longOpDumpTimeout);
}
+ ctx.performance().add("Disable assertions (remove '-ea' from JVM options)", !U.assertionsEnabled());
+
ctx.performance().logSuggestions(log, gridName);
U.quietAndInfo(log, "To start Console Management & Monitoring run ignitevisorcmd.{sh|bat}");
@@ -3509,6 +3544,26 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
}
+ /**
+ * @param node Node.
+ * @param payload Message payload.
+ * @param procFromNioThread If {@code true} message is processed from NIO thread.
+ * @return Response future.
+ */
+ public IgniteInternalFuture sendIoTest(ClusterNode node, byte[] payload, boolean procFromNioThread) {
+ return ctx.io().sendIoTest(node, payload, procFromNioThread);
+ }
+
+ /**
+ * @param nodes Nodes.
+ * @param payload Message payload.
+ * @param procFromNioThread If {@code true} message is processed from NIO thread.
+ * @return Response future.
+ */
+ public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload, boolean procFromNioThread) {
+ return ctx.io().sendIoTest(nodes, payload, procFromNioThread);
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(IgniteKernal.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index b3a9eec..f32a753 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -66,6 +66,7 @@ import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.StripedExecutor;
import org.apache.ignite.internal.util.spring.IgniteSpringHelper;
import org.apache.ignite.internal.util.typedef.CA;
import org.apache.ignite.internal.util.typedef.F;
@@ -1459,6 +1460,9 @@ public class IgnitionEx {
/** System executor service. */
private ThreadPoolExecutor sysExecSvc;
+ /** */
+ private StripedExecutor stripedExecSvc;
+
/** Management executor service. */
private ThreadPoolExecutor mgmtExecSvc;
@@ -1652,8 +1656,6 @@ public class IgnitionEx {
execSvc.allowCoreThreadTimeOut(true);
- // Note that since we use 'LinkedBlockingQueue', number of
- // maximum threads has no effect.
validateThreadPoolSize(cfg.getSystemThreadPoolSize(), "system");
sysExecSvc = new IgniteThreadPoolExecutor(
@@ -1666,6 +1668,9 @@ public class IgnitionEx {
sysExecSvc.allowCoreThreadTimeOut(true);
+ if (cfg.getStripedPoolSize() > 0)
+ stripedExecSvc = new StripedExecutor(cfg.getStripedPoolSize(), cfg.getGridName(), "sys", log);
+
// Note that since we use 'LinkedBlockingQueue', number of
// maximum threads has no effect.
// Note, that we do not pre-start threads here as management pool may
@@ -1791,13 +1796,26 @@ public class IgnitionEx {
// Init here to make grid available to lifecycle listeners.
grid = grid0;
- grid0.start(myCfg, utilityCacheExecSvc, marshCacheExecSvc, execSvc, sysExecSvc, p2pExecSvc, mgmtExecSvc,
- igfsExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc,
+ grid0.start(
+ myCfg,
+ utilityCacheExecSvc,
+ marshCacheExecSvc,
+ execSvc,
+ sysExecSvc,
+ stripedExecSvc,
+ p2pExecSvc,
+ mgmtExecSvc,
+ igfsExecSvc,
+ restExecSvc,
+ affExecSvc,
+ idxExecSvc,
+ callbackExecSvc,
new CA() {
@Override public void apply() {
startLatch.countDown();
}
- });
+ }
+ );
state = STARTED;
@@ -2415,6 +2433,10 @@ public class IgnitionEx {
sysExecSvc = null;
+ U.shutdownNow(getClass(), stripedExecSvc, log);
+
+ stripedExecSvc = null;
+
U.shutdownNow(getClass(), mgmtExecSvc, log);
mgmtExecSvc = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 6ea7c22..12baee0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -200,6 +200,7 @@ public abstract class GridClientConnectionManagerAdapter implements GridClientCo
.socketSendBufferSize(0)
.idleTimeout(Long.MAX_VALUE)
.gridName(routerClient ? "routerClient" : "gridClient")
+ .serverName("tcp-client")
.daemon(cfg.isDaemon())
.build();
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
index 06a4929..3566830 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterImpl.java
@@ -258,6 +258,7 @@ public class GridTcpRouterImpl implements GridTcpRouter, GridTcpRouterMBean, Lif
.logger(log)
.selectorCount(Runtime.getRuntime().availableProcessors())
.gridName(gridName)
+ .serverName("router")
.tcpNoDelay(tcpNoDelay)
.directBuffer(false)
.byteOrder(ByteOrder.nativeOrder())
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 3df29cf..7ef7bc0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -26,15 +26,17 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
@@ -44,6 +46,7 @@ import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.GridTopic;
import org.apache.ignite.internal.IgniteComponentType;
import org.apache.ignite.internal.IgniteDeploymentCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.direct.DirectMessageReader;
import org.apache.ignite.internal.direct.DirectMessageWriter;
import org.apache.ignite.internal.managers.GridManagerAdapter;
@@ -55,6 +58,8 @@ import org.apache.ignite.internal.processors.pool.PoolProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
import org.apache.ignite.internal.util.GridSpinReadWriteLock;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.lang.GridTuple3;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
@@ -83,6 +88,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.GridTopic.TOPIC_COMM_USER;
+import static org.apache.ignite.internal.GridTopic.TOPIC_IO_TEST;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.AFFINITY_POOL;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IDX_POOL;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.IGFS_POOL;
@@ -176,6 +182,12 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
/** Stopping flag. */
private boolean stopping;
+ /** */
+ private final AtomicReference<ConcurrentHashMap<Long, IoTestFuture>> ioTestMap = new AtomicReference<>();
+
+ /** */
+ private final AtomicLong ioTestId = new AtomicLong();
+
/**
* @param ctx Grid kernal context.
*/
@@ -297,6 +309,114 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
if (log.isDebugEnabled())
log.debug(startInfo());
+
+ addMessageListener(GridTopic.TOPIC_IO_TEST, new GridMessageListener() {
+ @Override public void onMessage(UUID nodeId, Object msg) {
+ ClusterNode node = ctx.discovery().node(nodeId);
+
+ if (node == null)
+ return;
+
+ IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg;
+
+ if (msg0.request()) {
+ IgniteIoTestMessage res = new IgniteIoTestMessage(msg0.id(), false, null);
+
+ res.flags(msg0.flags());
+
+ try {
+ send(node, GridTopic.TOPIC_IO_TEST, res, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send IO test response [msg=" + msg0 + "]", e);
+ }
+ }
+ else {
+ IoTestFuture fut = ioTestMap().get(msg0.id());
+
+ if (fut == null)
+ U.warn(log, "Failed to find IO test future [msg=" + msg0 + ']');
+ else
+ fut.onResponse();
+ }
+ }
+ });
+ }
+
+ /**
+ * @param nodes Nodes.
+ * @param payload Payload.
+ * @param procFromNioThread If {@code true} message is processed from NIO thread.
+ * @return Response future.
+ */
+ public IgniteInternalFuture sendIoTest(List<ClusterNode> nodes, byte[] payload, boolean procFromNioThread) {
+ long id = ioTestId.getAndIncrement();
+
+ IoTestFuture fut = new IoTestFuture(id, nodes.size());
+
+ IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+ msg.processFromNioThread(procFromNioThread);
+
+ ioTestMap().put(id, fut);
+
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+
+ try {
+ send(node, GridTopic.TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (IgniteCheckedException e) {
+ ioTestMap().remove(msg.id());
+
+ return new GridFinishedFuture(e);
+ }
+ }
+
+ return fut;
+ }
+
+ /**
+ * @param node Node.
+ * @param payload Payload.
+ * @param procFromNioThread If {@code true} message is processed from NIO thread.
+ * @return Response future.
+ */
+ public IgniteInternalFuture sendIoTest(ClusterNode node, byte[] payload, boolean procFromNioThread) {
+ long id = ioTestId.getAndIncrement();
+
+ IoTestFuture fut = new IoTestFuture(id, 1);
+
+ IgniteIoTestMessage msg = new IgniteIoTestMessage(id, true, payload);
+
+ msg.processFromNioThread(procFromNioThread);
+
+ ioTestMap().put(id, fut);
+
+ try {
+ send(node, GridTopic.TOPIC_IO_TEST, msg, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (IgniteCheckedException e) {
+ ioTestMap().remove(msg.id());
+
+ return new GridFinishedFuture(e);
+ }
+
+ return fut;
+ }
+
+ /**
+ * @return IO test futures map.
+ */
+ private ConcurrentHashMap<Long, IoTestFuture> ioTestMap() {
+ ConcurrentHashMap<Long, IoTestFuture> map = ioTestMap.get();
+
+ if (map == null) {
+ if (!ioTestMap.compareAndSet(null, map = new ConcurrentHashMap<>()))
+ map = ioTestMap.get();
+ }
+
+ return map;
}
/** {@inheritDoc} */
@@ -514,16 +634,6 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
return;
}
- // Check discovery.
- ClusterNode node = ctx.discovery().node(nodeId);
-
- if (node == null) {
- if (log.isDebugEnabled())
- log.debug("Ignoring message from dead node [senderId=" + nodeId + ", msg=" + msg + ']');
-
- return; // We can't receive messages from non-discovered ones.
- }
-
if (msg.topic() == null) {
int topicOrd = msg.topicOrdinal();
@@ -678,8 +788,31 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
msgC.run();
}
}
+
+ @Override public String toString() {
+ return "Message closure [msg=" + msg + ']';
+ }
};
+ if (msg.topicOrdinal() == TOPIC_IO_TEST.ordinal()) {
+ IgniteIoTestMessage msg0 = (IgniteIoTestMessage)msg.message();
+
+ if (msg0.processFromNioThread()) {
+ c.run();
+
+ return;
+ }
+ }
+
+ if (ctx.config().getStripedPoolSize() > 0 &&
+ plc == GridIoPolicy.SYSTEM_POOL &&
+ msg.partition() != Integer.MIN_VALUE
+ ) {
+ ctx.getStripedExecutorService().execute(msg.partition(), c);
+
+ return;
+ }
+
try {
pools.poolForPolicy(plc).execute(c);
}
@@ -2460,4 +2593,56 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
return S.toString(DelayedMessage.class, this, super.toString());
}
}
+
+ /**
+ *
+ */
+ private class IoTestFuture extends GridFutureAdapter<Object> {
+ /** */
+ private final long id;
+
+ /** */
+ private int cntr;
+
+ /**
+ * @param id ID.
+ * @param cntr Counter.
+ */
+ IoTestFuture(long id, int cntr) {
+ assert cntr > 0 : cntr;
+
+ this.id = id;
+ this.cntr = cntr;
+ }
+
+ /**
+ *
+ */
+ void onResponse() {
+ boolean complete;
+
+ synchronized (this) {
+ complete = --cntr == 0;
+ }
+
+ if (complete)
+ onDone();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable Object res, @Nullable Throwable err) {
+ if (super.onDone(res, err)) {
+ ioTestMap().remove(id);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IoTestFuture.class, this);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
index b28ced2..b1a26e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessage.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.managers.communication;
import java.io.Externalizable;
import java.nio.ByteBuffer;
import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.cache.GridCacheMessage;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.plugin.extensions.communication.Message;
@@ -321,6 +322,18 @@ public class GridIoMessage implements Message {
return 7;
}
+ /**
+ * Get single partition for this message (if applicable).
+ *
+ * @return Partition ID.
+ */
+ public int partition() {
+ if (msg instanceof GridCacheMessage)
+ return ((GridCacheMessage)msg).partition();
+ else
+ return Integer.MIN_VALUE;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridIoMessage.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 504e683..b1fe910 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -171,6 +171,16 @@ public class GridIoMessageFactory implements MessageFactory {
Message msg = null;
switch (type) {
+ case -44:
+ msg = new TcpCommunicationSpi.HandshakeMessage2();
+
+ break;
+
+ case -43:
+ msg = new IgniteIoTestMessage();
+
+ break;
+
case -42:
msg = new HadoopDirectShuffleMessage();
@@ -816,7 +826,7 @@ public class GridIoMessageFactory implements MessageFactory {
break;
- // [-3..119] [124..127] [-36]- this
+ // [-3..119] [124..127] [-36..-44]- this
// [120..123] - DR
// [-4..-22, -30..-35] - SQL
default:
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
new file mode 100644
index 0000000..77aaa09
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.communication;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+import java.nio.ByteBuffer;
+
+/**
+ *
+ */
+public class IgniteIoTestMessage implements Message {
+ /** */
+ private static byte FLAG_PROC_FROM_NIO = 1;
+
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private long id;
+
+ /** */
+ private byte flags;
+
+ /** */
+ private boolean req;
+
+ /** */
+ private byte payload[];
+
+ /**
+ *
+ */
+ IgniteIoTestMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param id Message ID.
+ * @param req Request flag.
+ * @param payload Payload.
+ */
+ IgniteIoTestMessage(long id, boolean req, byte[] payload) {
+ this.id = id;
+ this.req = req;
+ this.payload = payload;
+ }
+
+ /**
+ * @return {@code True} if message should be processed from NIO thread
+ * (otherwise message is submitted to system pool).
+ */
+ boolean processFromNioThread() {
+ return isFlag(FLAG_PROC_FROM_NIO);
+ }
+
+ /**
+ * @param procFromNioThread {@code True} if message should be processed from NIO thread.
+ */
+ void processFromNioThread(boolean procFromNioThread) {
+ setFlag(procFromNioThread, FLAG_PROC_FROM_NIO);
+ }
+
+ /**
+ * @param flags Flags.
+ */
+ public void flags(byte flags) {
+ this.flags = flags;
+ }
+
+ /**
+ * @return Flags.
+ */
+ public byte flags() {
+ return flags;
+ }
+
+ /**
+ * Sets flag mask.
+ *
+ * @param flag Set or clear.
+ * @param mask Mask.
+ */
+ private void setFlag(boolean flag, int mask) {
+ flags = flag ? (byte)(flags | mask) : (byte)(flags & ~mask);
+ }
+
+ /**
+ * Reads flag mask.
+ *
+ * @param mask Mask to read.
+ * @return Flag value.
+ */
+ private boolean isFlag(int mask) {
+ return (flags & mask) != 0;
+ }
+
+ /**
+ * @return {@code true} if this is request.
+ */
+ public boolean request() {
+ return req;
+ }
+
+ /**
+ * @return ID.
+ */
+ public long id() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeByte("flags", flags))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeLong("id", id))
+ return false;
+
+ writer.incrementState();
+
+ case 2:
+ if (!writer.writeByteArray("payload", payload))
+ return false;
+
+ writer.incrementState();
+
+ case 3:
+ if (!writer.writeBoolean("req", req))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ flags = reader.readByte("flags");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ id = reader.readLong("id");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 2:
+ payload = reader.readByteArray("payload");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 3:
+ req = reader.readBoolean("req");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(IgniteIoTestMessage.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return -43;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgniteIoTestMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 2e24e67..a8d9f1d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -288,6 +288,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** Asynchronous operations limit semaphore. */
private Semaphore asyncOpsSem;
+ /** */
+ protected volatile boolean asyncToggled;
+
/** {@inheritDoc} */
@Override public String name() {
return cacheCfg.getName();
@@ -364,6 +367,18 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
/**
+ * Toggles async flag if someone calls {@code withAsync()}
+ * on proxy and since that we have to properly handle all cache
+ * operations (sync and async) to put them in proper sequence.
+ *
+ * TODO: https://issues.apache.org/jira/browse/IGNITE-4393
+ */
+ void toggleAsync() {
+ if (!asyncToggled)
+ asyncToggled = true;
+ }
+
+ /**
* Prints memory stats.
*/
public void printMemoryStats() {
@@ -1134,7 +1149,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
execSvc = Executors.newFixedThreadPool(jobs.size() - 1);
for (int i = 1; i < jobs.size(); i++)
- execSvc.submit(jobs.get(i));
+ execSvc.execute(jobs.get(i));
}
try {
@@ -2534,6 +2549,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @return Put future.
*/
public IgniteInternalFuture<Boolean> putAsync(K key, V val, @Nullable CacheEntryPredicate filter) {
+ A.notNull(key, "key", val, "val");
+
final boolean statsEnabled = ctx.config().isStatisticsEnabled();
final long start = statsEnabled ? System.nanoTime() : 0L;
@@ -2554,8 +2571,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
*/
public IgniteInternalFuture<Boolean> putAsync0(final K key, final V val,
@Nullable final CacheEntryPredicate filter) {
- A.notNull(key, "key", val, "val");
-
if (keyCheck)
validateCacheKey(key);
@@ -4592,6 +4607,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* @return Failed future if waiting was interrupted.
*/
@Nullable protected <T> IgniteInternalFuture<T> asyncOpAcquire() {
+ if (!asyncToggled)
+ return null;
+
try {
if (asyncOpsSem != null)
asyncOpsSem.acquire();
@@ -4610,7 +4628,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
* Releases asynchronous operations permit, if limited.
*/
protected void asyncOpRelease() {
- if (asyncOpsSem != null)
+ if (asyncOpsSem != null && asyncToggled)
asyncOpsSem.release();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
index 71f99d3..0646d5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
@@ -129,6 +129,13 @@ public abstract class GridCacheMessage implements Message {
}
/**
+ * @return Partition ID this message is targeted to or {@code -1} if it cannot be determined.
+ */
+ public int partition() {
+ return -1;
+ }
+
+ /**
* If class loading error occurred during unmarshalling and {@link #ignoreClassErrors()} is
* set to {@code true}, then the error will be passed into this method.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 90e428c..3178203 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -110,6 +110,41 @@ import static org.apache.ignite.internal.processors.cache.GridCacheOperation.REA
* Cache utility methods.
*/
public class GridCacheUtils {
+ /** Cheat cache ID for debugging and benchmarking purposes. */
+ public static final int cheatCacheId;
+
+ /*
+ *
+ */
+ static {
+ String cheatCache = System.getProperty("CHEAT_CACHE");
+
+ if (cheatCache != null) {
+ cheatCacheId = cheatCache.hashCode();
+
+ if (cheatCacheId == 0)
+ throw new RuntimeException();
+
+ System.out.println(">>> Cheat cache ID [id=" + cheatCacheId + ", name=" + cheatCache + ']');
+ }
+ else
+ cheatCacheId = 0;
+ }
+
+ /**
+ * Quickly checks if passed in cache ID is a "cheat cache ID" set by -DCHEAT_CACHE=user_cache_name
+ * and resolved in static block above.
+ *
+ * FOR DEBUGGING AND TESTING PURPOSES!
+ *
+ * @param id Cache ID to check.
+ * @return {@code True} if this is cheat cache ID.
+ */
+ @Deprecated
+ public static boolean cheatCache(int id) {
+ return cheatCacheId != 0 && id == cheatCacheId;
+ }
+
/** Hadoop syste cache name. */
public static final String SYS_CACHE_HADOOP_MR = "ignite-hadoop-mr-sys-cache";
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index f87fa1d..b9e6e82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -334,6 +334,14 @@ public class IgniteCacheProxy<K, V> extends AsyncSupportAdapter<IgniteCache<K, V
}
/** {@inheritDoc} */
+ @Override public IgniteCache<K, V> withAsync() {
+ if (delegate instanceof GridCacheAdapter)
+ ((GridCacheAdapter)delegate).toggleAsync();
+
+ return super.withAsync();
+ }
+
+ /** {@inheritDoc} */
@Override public IgniteCache<K, V> withSkipStore() {
return skipStore();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
index 9639a9a..a671296 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
@@ -310,6 +310,11 @@ public class GridDistributedLockRequest extends GridDistributedBaseMessage {
return keys;
}
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+ }
+
/**
* @return Max lock wait time.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
index 109d665..c5cf332 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
@@ -86,6 +86,12 @@ public class GridDistributedTxFinishResponse extends GridCacheMessage {
}
/** {@inheritDoc} */
+ @Override public int partition() {
+ // TODO https://issues.apache.org/jira/browse/IGNITE-4371
+ return Integer.MIN_VALUE;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
writer.setBuffer(buf);
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
index df6acdd..5d70ec1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
@@ -89,6 +89,11 @@ public class GridDistributedUnlockRequest extends GridDistributedBaseMessage {
partIds.add(key.partition());
}
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+ }
+
/** {@inheritDoc}
* @param ctx*/
@Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 35e6267..519d0fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -218,7 +218,8 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
@Override public void onKernalStart() throws IgniteCheckedException {
super.onKernalStart();
- preldr.onKernalStart();
+ if (preldr != null)
+ preldr.onKernalStart();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
index 1e92b54..63e3309 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.GridDirectCollection;
@@ -31,7 +32,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockResponse;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.GridLeanSet;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteUuid;
@@ -57,7 +57,7 @@ public class GridDhtLockResponse extends GridDistributedLockResponse {
/** Invalid partitions. */
@GridToStringInclude
@GridDirectCollection(int.class)
- private Collection<Integer> invalidParts = new GridLeanSet<>();
+ private Collection<Integer> invalidParts;
/** Preload entries. */
@GridDirectCollection(GridCacheEntryInfo.class)
@@ -127,6 +127,9 @@ public class GridDhtLockResponse extends GridDistributedLockResponse {
* @param part Invalid partition.
*/
public void addInvalidPartition(int part) {
+ if (invalidParts == null)
+ invalidParts = new HashSet<>();
+
invalidParts.add(part);
}
@@ -134,7 +137,7 @@ public class GridDhtLockResponse extends GridDistributedLockResponse {
* @return Invalid partitions.
*/
public Collection<Integer> invalidPartitions() {
- return invalidParts;
+ return invalidParts == null ? Collections.<Integer>emptySet() : invalidParts;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 940c74e..0e60ff4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -613,8 +613,6 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public IgniteInternalFuture<Boolean> putAsync0(K key, V val, @Nullable CacheEntryPredicate filter) {
- A.notNull(key, "key", val, "val");
-
return updateAsync0(
key,
val,
@@ -814,6 +812,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
*/
@SuppressWarnings("unchecked")
protected <T> IgniteInternalFuture<T> asyncOp(final CO<IgniteInternalFuture<T>> op) {
+ if (!asyncToggled)
+ return op.apply();
+
IgniteInternalFuture<T> fail = asyncOpAcquire();
if (fail != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
index a03d948..0af7cf5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
@@ -235,6 +235,11 @@ public class GridDhtAtomicSingleUpdateRequest extends GridDhtAtomicAbstractUpdat
}
/** {@inheritDoc} */
+ @Override public int partition() {
+ return partId;
+ }
+
+ /** {@inheritDoc} */
@Override public int partitionId(int idx) {
assert idx == 0 : idx;
http://git-wip-us.apache.org/repos/asf/ignite/blob/05dd08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
index f2fbb0e..1854e52 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
@@ -487,6 +487,11 @@ public class GridDhtAtomicUpdateRequest extends GridDhtAtomicAbstractUpdateReque
}
/** {@inheritDoc} */
+ @Override public int partition() {
+ return partIds != null && !partIds.isEmpty() ? partIds.get(0) : -1;
+ }
+
+ /** {@inheritDoc} */
@Override public long conflictExpireTime(int idx) {
if (conflictExpireTimes != null) {
assert idx >= 0 && idx < conflictExpireTimes.size();
[21/50] [abbrv] ignite git commit: IGNITE-4281: Hadoop: decoupled
remote and local maps to simplify further optimizations. This closes #1264.
This closes #1315.
Posted by sb...@apache.org.
IGNITE-4281: Hadoop: decoupled remote and local maps to simplify further optimizations. This closes #1264. This closes #1315.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/be12a7ea
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/be12a7ea
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/be12a7ea
Branch: refs/heads/master
Commit: be12a7ea242dedba932c15dce005540c34711e77
Parents: 04cff9b
Author: devozerov <vo...@gridgain.com>
Authored: Mon Dec 5 17:09:28 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:01 2016 +0300
----------------------------------------------------------------------
.../hadoop/shuffle/HadoopShuffleJob.java | 85 ++++++++++++++------
1 file changed, 61 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/be12a7ea/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
index aca5fdf..3afb55a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleJob.java
@@ -17,9 +17,7 @@
package org.apache.ignite.internal.processors.hadoop.shuffle;
-import java.util.HashMap;
import java.util.Iterator;
-import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@@ -56,8 +54,8 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.thread.IgniteThread;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.PARTITION_HASHMAP_SIZE;
-import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_JOB_THROTTLE;
+import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_MSG_SIZE;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.SHUFFLE_REDUCER_NO_SORTING;
import static org.apache.ignite.internal.processors.hadoop.HadoopJobProperty.get;
@@ -77,20 +75,26 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
/** */
private final boolean needPartitioner;
- /** Collection of task contexts for each reduce task. */
- private final Map<Integer, LocalTaskContextProxy> reducersCtx = new HashMap<>();
+ /** Task contexts for each reduce task. */
+ private final AtomicReferenceArray<LocalTaskContextProxy> locReducersCtx;
/** Reducers addresses. */
private T[] reduceAddrs;
+ /** Total reducer count. */
+ private final int totalReducerCnt;
+
/** Local reducers address. */
private final T locReduceAddr;
/** */
private final HadoopShuffleMessage[] msgs;
- /** */
- private final AtomicReferenceArray<HadoopMultimap> maps;
+ /** Maps for local reducers. */
+ private final AtomicReferenceArray<HadoopMultimap> locMaps;
+
+ /** Maps for remote reducers. */
+ private final AtomicReferenceArray<HadoopMultimap> rmtMaps;
/** */
private volatile IgniteInClosure2X<T, HadoopMessage> io;
@@ -129,23 +133,27 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
public HadoopShuffleJob(T locReduceAddr, IgniteLogger log, HadoopJob job, GridUnsafeMemory mem,
int totalReducerCnt, int[] locReducers) throws IgniteCheckedException {
this.locReduceAddr = locReduceAddr;
+ this.totalReducerCnt = totalReducerCnt;
this.job = job;
this.mem = mem;
this.log = log.getLogger(HadoopShuffleJob.class);
msgSize = get(job.info(), SHUFFLE_MSG_SIZE, DFLT_SHUFFLE_MSG_SIZE);
+ locReducersCtx = new AtomicReferenceArray<>(totalReducerCnt);
+
if (!F.isEmpty(locReducers)) {
for (int rdc : locReducers) {
HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, job.id(), rdc, 0, null);
- reducersCtx.put(rdc, new LocalTaskContextProxy(taskInfo));
+ locReducersCtx.set(rdc, new LocalTaskContextProxy(taskInfo));
}
}
needPartitioner = totalReducerCnt > 1;
- maps = new AtomicReferenceArray<>(totalReducerCnt);
+ locMaps = new AtomicReferenceArray<>(totalReducerCnt);
+ rmtMaps = new AtomicReferenceArray<>(totalReducerCnt);
msgs = new HadoopShuffleMessage[totalReducerCnt];
throttle = get(job.info(), SHUFFLE_JOB_THROTTLE, 0);
@@ -237,13 +245,13 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
assert msg.buffer() != null;
assert msg.offset() > 0;
- HadoopTaskContext taskCtx = reducersCtx.get(msg.reducer()).get();
+ HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
- HadoopMultimap map = getOrCreateMap(maps, msg.reducer());
+ HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
// Add data from message to the map.
try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
@@ -320,10 +328,10 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
* Sends map updates to remote reducers.
*/
private void collectUpdatesAndSend(boolean flush) throws IgniteCheckedException {
- for (int i = 0; i < maps.length(); i++) {
- HadoopMultimap map = maps.get(i);
+ for (int i = 0; i < rmtMaps.length(); i++) {
+ HadoopMultimap map = rmtMaps.get(i);
- if (map == null || locReduceAddr.equals(reduceAddrs[i]))
+ if (map == null)
continue; // Skip empty map and local node.
if (msgs[i] == null)
@@ -448,7 +456,8 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
}
- close(maps);
+ close(locMaps);
+ close(rmtMaps);
}
/**
@@ -473,7 +482,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
flushed = true;
- if (maps.length() == 0)
+ if (totalReducerCnt == 0)
return new GridFinishedFuture<>();
U.await(ioInitLatch);
@@ -544,7 +553,7 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
case REDUCE:
int reducer = taskCtx.taskInfo().taskNumber();
- HadoopMultimap m = maps.get(reducer);
+ HadoopMultimap m = locMaps.get(reducer);
if (m != null)
return m.input(taskCtx);
@@ -573,11 +582,24 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
}
/**
+ * Check if certain partition (reducer) is local.
+ *
+ * @param part Partition.
+ * @return {@code True} if local.
+ */
+ private boolean isLocalPartition(int part) {
+ return locReducersCtx.get(part) != null;
+ }
+
+ /**
* Partitioned output.
*/
private class PartitionedOutput implements HadoopTaskOutput {
/** */
- private final HadoopTaskOutput[] adders = new HadoopTaskOutput[maps.length()];
+ private final HadoopTaskOutput[] locAdders = new HadoopTaskOutput[locMaps.length()];
+
+ /** */
+ private final HadoopTaskOutput[] rmtAdders = new HadoopTaskOutput[rmtMaps.length()];
/** */
private HadoopPartitioner partitioner;
@@ -601,23 +623,38 @@ public class HadoopShuffleJob<T> implements AutoCloseable {
int part = 0;
if (partitioner != null) {
- part = partitioner.partition(key, val, adders.length);
+ part = partitioner.partition(key, val, totalReducerCnt);
- if (part < 0 || part >= adders.length)
+ if (part < 0 || part >= totalReducerCnt)
throw new IgniteCheckedException("Invalid partition: " + part);
}
- HadoopTaskOutput out = adders[part];
+ HadoopTaskOutput out;
- if (out == null)
- adders[part] = out = getOrCreateMap(maps, part).startAdding(taskCtx);
+ if (isLocalPartition(part)) {
+ out = locAdders[part];
+
+ if (out == null)
+ locAdders[part] = out = getOrCreateMap(locMaps, part).startAdding(taskCtx);
+ }
+ else {
+ out = rmtAdders[part];
+
+ if (out == null)
+ rmtAdders[part] = out = getOrCreateMap(rmtMaps, part).startAdding(taskCtx);
+ }
out.write(key, val);
}
/** {@inheritDoc} */
@Override public void close() throws IgniteCheckedException {
- for (HadoopTaskOutput adder : adders) {
+ for (HadoopTaskOutput adder : locAdders) {
+ if (adder != null)
+ adder.close();
+ }
+
+ for (HadoopTaskOutput adder : rmtAdders) {
if (adder != null)
adder.close();
}
[47/50] [abbrv] ignite git commit: ignite-4379: fixed issue after
merge with latest DML changes
Posted by sb...@apache.org.
ignite-4379: fixed issue after merge with latest DML changes
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e8fd1657
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e8fd1657
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e8fd1657
Branch: refs/heads/master
Commit: e8fd1657de7847922feec3d3e58250ad116e4e47
Parents: 800579c
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 20 15:27:04 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 20 15:27:04 2016 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/query/h2/IgniteH2Indexing.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8fd1657/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index c541185..9037943 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -801,8 +801,6 @@ public class IgniteH2Indexing implements GridQueryIndexing {
Prepared p = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
if (!p.isQuery()) {
- GridH2QueryContext.clearThreadLocal();
-
SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry);
if (params != null)
[26/50] [abbrv] ignite git commit: IGNITE-4341: Hadoop: added
Terasort to unit tests. This closes #1302. This closes #1321.
Posted by sb...@apache.org.
IGNITE-4341: Hadoop: added Terasort to unit tests. This closes #1302. This closes #1321.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b44baf1e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b44baf1e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b44baf1e
Branch: refs/heads/master
Commit: b44baf1e8c42c57fa4e241d5943593fa4ae42f12
Parents: ffe53eb
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Dec 12 16:52:47 2016 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 15 13:46:29 2016 +0300
----------------------------------------------------------------------
modules/hadoop/pom.xml | 7 +
.../hadoop/impl/fs/HadoopFileSystemsUtils.java | 11 +
.../impl/v2/HadoopV2JobResourceManager.java | 25 +-
.../hadoop/impl/HadoopAbstractSelfTest.java | 13 +-
.../impl/HadoopAbstractWordCountTest.java | 6 +-
.../hadoop/impl/HadoopFileSystemsTest.java | 9 +
.../hadoop/impl/HadoopJobTrackerSelfTest.java | 4 +-
.../impl/HadoopTaskExecutionSelfTest.java | 4 +-
.../hadoop/impl/HadoopTeraSortTest.java | 376 +++++++++++++++++++
.../client/HadoopClientProtocolSelfTest.java | 4 +-
.../collections/HadoopSkipListSelfTest.java | 14 +-
.../HadoopExternalTaskExecutionSelfTest.java | 2 +
.../testsuites/IgniteHadoopTestSuite.java | 3 +
13 files changed, 450 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index d0b0481..db302d7 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -95,6 +95,13 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-examples</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.gridgain</groupId>
<artifactId>ignite-shmem</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
index 5115cb4..37902f0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/fs/HadoopFileSystemsUtils.java
@@ -17,7 +17,9 @@
package org.apache.ignite.internal.processors.hadoop.impl.fs;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.jetbrains.annotations.Nullable;
@@ -48,4 +50,13 @@ public class HadoopFileSystemsUtils {
public static String disableFsCachePropertyName(@Nullable String scheme) {
return String.format("fs.%s.impl.disable.cache", scheme);
}
+
+ /**
+ * Clears Hadoop {@link FileSystem} cache.
+ *
+ * @throws IOException On error.
+ */
+ public static void clearFileSystemCache() throws IOException {
+ FileSystem.closeAll();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
index 3984f83..52e394b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
@@ -123,7 +123,9 @@ class HadoopV2JobResourceManager {
JobConf cfg = ctx.getJobConf();
- String mrDir = cfg.get("mapreduce.job.dir");
+ Collection<URL> clsPathUrls = new ArrayList<>();
+
+ String mrDir = cfg.get(MRJobConfig.MAPREDUCE_JOB_DIR);
if (mrDir != null) {
stagingDir = new Path(new URI(mrDir));
@@ -144,28 +146,23 @@ class HadoopV2JobResourceManager {
File jarJobFile = new File(jobLocDir, "job.jar");
- Collection<URL> clsPathUrls = new ArrayList<>();
-
clsPathUrls.add(jarJobFile.toURI().toURL());
rsrcSet.add(jarJobFile);
rsrcSet.add(new File(jobLocDir, "job.xml"));
-
- processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
- processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES);
- processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
- processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);
-
- if (!clsPathUrls.isEmpty()) {
- clsPath = new URL[clsPathUrls.size()];
-
- clsPathUrls.toArray(clsPath);
- }
}
else if (!jobLocDir.mkdirs())
throw new IgniteCheckedException("Failed to create local job directory: "
+ jobLocDir.getAbsolutePath());
+ processFiles(jobLocDir, ctx.getCacheFiles(), download, false, null, MRJobConfig.CACHE_LOCALFILES);
+ processFiles(jobLocDir, ctx.getCacheArchives(), download, true, null, MRJobConfig.CACHE_LOCALARCHIVES);
+ processFiles(jobLocDir, ctx.getFileClassPaths(), download, false, clsPathUrls, null);
+ processFiles(jobLocDir, ctx.getArchiveClassPaths(), download, true, clsPathUrls, null);
+
+ if (!clsPathUrls.isEmpty())
+ clsPath = clsPathUrls.toArray(new URL[clsPathUrls.size()]);
+
setLocalFSWorkingDirectory(jobLocDir);
}
catch (URISyntaxException | IOException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
index 12351c6..5666cbc 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractSelfTest.java
@@ -83,7 +83,9 @@ public abstract class HadoopAbstractSelfTest extends GridCommonAbstractTest {
private static String initCp;
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
+ @Override protected final void beforeTestsStarted() throws Exception {
+ HadoopFileSystemsUtils.clearFileSystemCache();
+
// Add surefire classpath to regular classpath.
initCp = System.getProperty("java.class.path");
@@ -93,6 +95,15 @@ public abstract class HadoopAbstractSelfTest extends GridCommonAbstractTest {
System.setProperty("java.class.path", initCp + File.pathSeparatorChar + surefireCp);
super.beforeTestsStarted();
+
+ beforeTestsStarted0();
+ }
+
+ /**
+ * Performs additional initialization in the beginning of test class execution.
+ */
+ protected void beforeTestsStarted0() throws Exception {
+ // noop
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
index 3cb8f91..84e6aee 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopAbstractWordCountTest.java
@@ -49,9 +49,7 @@ public abstract class HadoopAbstractWordCountTest extends HadoopAbstractSelfTest
protected IgfsEx igfs;
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
+ @Override protected void beforeTestsStarted0() throws Exception {
Configuration cfg = new Configuration();
setupFileSystems(cfg);
@@ -62,6 +60,8 @@ public abstract class HadoopAbstractWordCountTest extends HadoopAbstractSelfTest
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
igfs = (IgfsEx)startGrids(gridCount()).fileSystem(igfsName);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
index 252d6cb..7680690 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopFileSystemsTest.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemsUtils;
+import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopLocalFileSystemV1;
import org.apache.ignite.testframework.GridTestUtils;
/**
@@ -37,11 +38,15 @@ public class HadoopFileSystemsTest extends HadoopAbstractSelfTest {
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
startGrids(gridCount());
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
+ super.afterTest();
+
stopAllGrids(true);
}
@@ -70,6 +75,10 @@ public class HadoopFileSystemsTest extends HadoopAbstractSelfTest {
cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP,
new Path(new Path(uri), "user/" + System.getProperty("user.name")).toString());
+ FileSystem fs = FileSystem.get(uri, cfg);
+
+ assertTrue(fs instanceof HadoopLocalFileSystemV1);
+
final CountDownLatch changeUserPhase = new CountDownLatch(THREAD_COUNT);
final CountDownLatch changeDirPhase = new CountDownLatch(THREAD_COUNT);
final CountDownLatch changeAbsDirPhase = new CountDownLatch(THREAD_COUNT);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
index a3bf49c..91ad5ec 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopJobTrackerSelfTest.java
@@ -65,9 +65,7 @@ public class HadoopJobTrackerSelfTest extends HadoopAbstractSelfTest {
}
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
+ @Override protected void beforeTestsStarted0() throws Exception {
startGrids(gridCount());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
index 027f921..9d45b03 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTaskExecutionSelfTest.java
@@ -83,9 +83,7 @@ public class HadoopTaskExecutionSelfTest extends HadoopAbstractSelfTest {
}
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
+ @Override protected void beforeTestsStarted0() throws Exception {
startGrids(gridCount());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
new file mode 100644
index 0000000..0cc9564
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTeraSortTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.util.UUID;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.terasort.TeraGen;
+import org.apache.hadoop.examples.terasort.TeraInputFormat;
+import org.apache.hadoop.examples.terasort.TeraOutputFormat;
+import org.apache.hadoop.examples.terasort.TeraSort;
+import org.apache.hadoop.examples.terasort.TeraValidate;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.HadoopConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
+
+import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.createJobInfo;
+
+/**
+ * Implements TeraSort Hadoop sample as a unit test.
+ */
+public class HadoopTeraSortTest extends HadoopAbstractSelfTest {
+ /** Copy of Hadoop constant of package-private visibility. */
+ public static final String PARTITION_FILENAME = getPartitionFileNameConstant();
+
+ /** Out destination dir. */
+ protected final String generateOutDir = getFsBase() + "/tera-generated";
+
+ /** Sort destination dir. */
+ protected final String sortOutDir = getFsBase() + "/tera-sorted";
+
+ /** Validation destination dir. */
+ protected final String validateOutDir = getFsBase() + "/tera-validated";
+
+ /**
+ * Extracts value of Hadoop package-private constant.
+ *
+ * @return TeraInputFormat.PARTITION_FILENAME.
+ */
+ private static String getPartitionFileNameConstant() {
+ try {
+ Field f = TeraInputFormat.class.getDeclaredField("PARTITION_FILENAME");
+
+ f.setAccessible(true);
+
+ return (String)f.get(null);
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /**
+ * Gets base directory.
+ * Note that this directory will be completely deleted in the and of the test.
+ * @return The base directory.
+ */
+ protected String getFsBase() {
+ return "file:///tmp/" + getUser() + "/hadoop-terasort-test";
+ }
+
+ /**
+ * @return Full input data size, in bytes.
+ */
+ protected long dataSizeBytes() {
+ return 100_000_000;
+ }
+
+ /**
+ * Desired number of maps in TeraSort job.
+ * @return The number of maps.
+ */
+ protected int numMaps() {
+ return gridCount() * 10;
+ }
+
+ /**
+ * Desired number of reduces in TeraSort job.
+ * @return The number of reduces.
+ */
+ protected int numReduces() {
+ return gridCount() * 8;
+ }
+
+ /**
+ * The user to run Hadoop job on behalf of.
+ * @return The user to run Hadoop job on behalf of.
+ */
+ protected String getUser() {
+ return System.getProperty("user.name");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+
+ // Delete files used:
+ getFileSystem().delete(new Path(getFsBase()), true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected final boolean igfsEnabled() {
+ return false;
+ }
+
+ /**
+ * Does actual test TeraSort job Through Ignite API
+ */
+ protected final void teraSort() throws Exception {
+ System.out.println("TeraSort ===============================================================");
+
+ getFileSystem().delete(new Path(sortOutDir), true);
+
+ final JobConf jobConf = new JobConf();
+
+ jobConf.setUser(getUser());
+
+ jobConf.set("fs.defaultFS", getFsBase());
+
+ log().info("Desired number of reduces: " + numReduces());
+
+ jobConf.set("mapreduce.job.reduces", String.valueOf(numReduces()));
+
+ log().info("Desired number of maps: " + numMaps());
+
+ final long splitSize = dataSizeBytes() / numMaps();
+
+ log().info("Desired split size: " + splitSize);
+
+ // Force the split to be of the desired size:
+ jobConf.set("mapred.min.split.size", String.valueOf(splitSize));
+ jobConf.set("mapred.max.split.size", String.valueOf(splitSize));
+
+ Job job = setupConfig(jobConf);
+
+ HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 1);
+
+ IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));
+
+ fut.get();
+ }
+
+ /**
+ * Gets the file system we work upon.
+ * @return The file system.
+ * @throws Exception
+ */
+ FileSystem getFileSystem() throws Exception{
+ return FileSystem.get(new URI(getFsBase()), new Configuration());
+ }
+
+ /**
+ * Represents the data generation stage.
+ * @throws Exception
+ */
+ private void teraGenerate() throws Exception {
+ System.out.println("TeraGenerate ===============================================================");
+
+ getFileSystem().delete(new Path(generateOutDir), true);
+
+ final long numLines = dataSizeBytes() / 100; // TeraGen makes 100 bytes ber line
+
+ if (numLines < 1)
+ throw new IllegalStateException("Data size is too small: " + dataSizeBytes());
+
+ // Generate input data:
+ int res = ToolRunner.run(new Configuration(), new TeraGen(), new String[] {"-Dmapreduce.framework.name=local",
+ String.valueOf(numLines), generateOutDir});
+
+ assertEquals(0, res);
+
+ FileStatus[] fileStatuses = getFileSystem().listStatus(new Path(generateOutDir));
+
+ long sumLen = 0;
+
+ for (FileStatus fs: fileStatuses)
+ sumLen += fs.getLen();
+
+ assertEquals(dataSizeBytes(), sumLen); // Ensure correct size data is generated.
+ }
+
+ /**
+ * Creates Job instance and sets up necessary properties for it.
+ * @param conf The Job config.
+ * @return The job.
+ * @throws Exception On error.
+ */
+ private Job setupConfig(JobConf conf) throws Exception {
+ Job job = Job.getInstance(conf);
+
+ Path inputDir = new Path(generateOutDir);
+ Path outputDir = new Path(sortOutDir);
+
+ boolean useSimplePartitioner = TeraSort.getUseSimplePartitioner(job);
+
+ TeraInputFormat.setInputPaths(job, inputDir);
+ FileOutputFormat.setOutputPath(job, outputDir);
+
+ job.setJobName("TeraSort");
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ job.setInputFormatClass(TeraInputFormat.class);
+ job.setOutputFormatClass(TeraOutputFormat.class);
+
+ if (useSimplePartitioner)
+ job.setPartitionerClass(TeraSort.SimplePartitioner.class);
+ else {
+ long start = System.currentTimeMillis();
+
+ Path partFile = new Path(outputDir, PARTITION_FILENAME);
+
+ URI partUri = new URI(partFile.toString() + "#" + PARTITION_FILENAME);
+
+ try {
+ TeraInputFormat.writePartitionFile(job, partFile);
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+
+ job.addCacheFile(partUri);
+
+ long end = System.currentTimeMillis();
+
+ System.out.println("Spent " + (end - start) + "ms computing partitions. " +
+ "Partition file added to distributed cache: " + partUri);
+
+ job.setPartitionerClass(getTeraSortTotalOrderPartitioner()/*TeraSort.TotalOrderPartitioner.class*/);
+ }
+
+ job.getConfiguration().setInt("dfs.replication", TeraSort.getOutputReplication(job));
+
+ /* TeraOutputFormat.setFinalSync(job, true); */
+ Method m = TeraOutputFormat.class.getDeclaredMethod("setFinalSync", JobContext.class, boolean.class);
+ m.setAccessible(true);
+ m.invoke(null, job, true);
+
+ return job;
+ }
+
+ /**
+ * Extracts package-private TeraSort total order partitioner class.
+ *
+ * @return The class.
+ */
+ @SuppressWarnings("unchecked")
+ private Class<? extends Partitioner> getTeraSortTotalOrderPartitioner() {
+ Class[] classes = TeraSort.class.getDeclaredClasses();
+
+ Class<? extends Partitioner> totalOrderPartitionerCls = null;
+
+ for (Class<?> x: classes) {
+ if ("TotalOrderPartitioner".equals(x.getSimpleName())) {
+ totalOrderPartitionerCls = (Class<? extends Partitioner>)x;
+
+ break;
+ }
+ }
+
+ if (totalOrderPartitionerCls == null)
+ throw new IllegalStateException("Failed to find TeraSort total order partitioner class.");
+
+ return totalOrderPartitionerCls;
+ }
+
+ /**
+ * Implements validation phase of the sample.
+ * @throws Exception
+ */
+ private void teraValidate() throws Exception {
+ System.out.println("TeraValidate ===============================================================");
+
+ getFileSystem().delete(new Path(validateOutDir), true);
+
+ // Generate input data:
+ int res = ToolRunner.run(new Configuration(), new TeraValidate(),
+ new String[] {"-Dmapreduce.framework.name=local", sortOutDir, validateOutDir});
+
+ assertEquals(0, res);
+
+ FileStatus[] fileStatuses = getFileSystem().listStatus(new Path(validateOutDir), new PathFilter() {
+ @Override public boolean accept(Path path) {
+ // Typically name is "part-r-00000":
+ return path.getName().startsWith("part-r-");
+ }
+ });
+
+ // TeraValidate has only 1 reduce, so should be only 1 result file:
+ assertEquals(1, fileStatuses.length);
+
+ // The result file must contain only 1 line with the checksum, like this:
+ // "checksum 7a27e2d0d55de",
+ // typically it has length of 23 bytes.
+ // If sorting was not correct, the result contains list of K-V pairs that are not ordered correctly.
+ // In such case the size of the output will be much larger.
+ long len = fileStatuses[0].getLen();
+
+ assertTrue("TeraValidate length: " + len, len >= 16 && len <= 32);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ getFileSystem().delete(new Path(getFsBase()), true);
+
+ startGrids(gridCount());
+ }
+
+ /**
+ * Runs generate/sort/validate phases of the terasort sample.
+ * @throws Exception
+ */
+ public void testTeraSort() throws Exception {
+ teraGenerate();
+
+ teraSort();
+
+ teraValidate();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration igc = super.getConfiguration(gridName);
+
+ HadoopConfiguration hc = createHadoopConfiguration();
+
+ igc.setHadoopConfiguration(hc);
+
+ return igc;
+ }
+
+ /**
+ * Creates Hadoop configuration for the test.
+ * @return The {@link HadoopConfiguration}.
+ */
+ protected HadoopConfiguration createHadoopConfiguration() {
+ HadoopConfiguration hadoopCfg = new HadoopConfiguration();
+
+ // See org.apache.ignite.configuration.HadoopConfiguration.DFLT_MAX_TASK_QUEUE_SIZE
+ hadoopCfg.setMaxTaskQueueSize(30_000);
+
+ return hadoopCfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
index 7156a3d..44fc46e 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/client/HadoopClientProtocolSelfTest.java
@@ -99,9 +99,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
}
/** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
+ @Override protected void beforeTestsStarted0() throws Exception {
startGrids(gridCount());
setupLockFile.delete();
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
index 111ea78..1138803 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/shuffle/collections/HadoopSkipListSelfTest.java
@@ -85,6 +85,9 @@ public class HadoopSkipListSelfTest extends HadoopAbstractMapTest {
}
}
+ /**
+ * @throws Exception On error.
+ */
public void testMapSimple() throws Exception {
GridUnsafeMemory mem = new GridUnsafeMemory(0);
@@ -139,7 +142,16 @@ public class HadoopSkipListSelfTest extends HadoopAbstractMapTest {
assertEquals(0, mem.allocatedSize());
}
- private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis, HadoopTaskContext taskCtx)
+ /**
+ * Check.
+ * @param m The multimap.
+ * @param mm The multimap storing expectations.
+ * @param vis The multimap to store visitor results.
+ * @param taskCtx The task context.
+ * @throws Exception On error.
+ */
+ private void check(HadoopMultimap m, Multimap<Integer, Integer> mm, final Multimap<Integer, Integer> vis,
+ HadoopTaskContext taskCtx)
throws Exception {
final HadoopTaskInput in = m.input(taskCtx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
index 7c43500..5f64ce7 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
@@ -57,6 +57,8 @@ public class HadoopExternalTaskExecutionSelfTest extends HadoopAbstractSelfTest
@Override protected void beforeTest() throws Exception {
fail("https://issues.apache.org/jira/browse/IGNITE-404");
+ super.beforeTest();
+
startGrids(gridCount());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b44baf1e/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 959bc59..6046cc1 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -23,6 +23,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopTeraSortTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolEmbeddedSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolMultipleServersSelfTest;
import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest;
@@ -123,6 +124,8 @@ public class IgniteHadoopTestSuite extends TestSuite {
suite.addTest(new TestSuite(ldr.loadClass(KerberosHadoopFileSystemFactorySelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(HadoopTeraSortTest.class.getName())));
+
suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyFullMapReduceTest.class.getName())));