You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ak...@apache.org on 2015/11/17 14:34:47 UTC
[09/46] ignite git commit: Performance optimizations.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
index 5c6cfcf..9fe2690 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
@@ -146,40 +146,41 @@ public class GridConcurrentLinkedHashMapMultiThreadedSelfTest extends GridCommon
public void testEvictPerSegment() throws Exception {
info(">>> Test grid concurrent linked hash map...");
- final int maxSize = 1000;
+ int concurLvl = 64;
+ final int maxSize = concurLvl * 30;
+ int diff = (int)(maxSize * 0.1);
ConcurrentLinkedHashMap<Integer, String> linkedMap = new ConcurrentLinkedHashMap<>(
- 32, 0.75f, 64, maxSize, PER_SEGMENT_Q);
+ 32, 0.75f, concurLvl, maxSize, PER_SEGMENT_Q);
int keyCnt = 1000000;
- putMultiThreaded(linkedMap, 10, keyCnt, maxSize);
-
- int diff = 10; // 1% of 1000.
+ Map<String, LinkedList<Integer>> map = putMultiThreaded(
+ linkedMap,
+ 10,
+ keyCnt,
+ maxSize * 10); // Intentionally memorize more than maxSize since in this mode LRU is not fair.
assertTrue("Invalid map size: " + linkedMap.size(), U.safeAbs(maxSize - linkedMap.size()) <= diff);
assertTrue("Invalid map sizex: " + linkedMap.sizex(), U.safeAbs(maxSize - linkedMap.sizex()) <= diff);
-// TODO IGNITE-606 - Need to fix iterators for ConcurrentLinkedHashMap in perSegment mode
-// LinkedList<Integer> keys = new LinkedList<Integer>(linkedMap.keySet());
-//
-// while (!keys.isEmpty()) {
-// boolean found = false;
-//
-// int key = keys.removeLast();
-//
-// for (LinkedList<Integer> threadKeys : map.values()) {
-// if (threadKeys.getLast() == key) {
-// threadKeys.removeLast();
-//
-// found = true;
-//
-// break;
-// }
-// }
-//
-// assertTrue("Key was not found on the top of any thread: " + key, found);
-// }
+ LinkedList<Integer> keys = new LinkedList<>(linkedMap.keySet());
+
+ while (!keys.isEmpty()) {
+ boolean found = false;
+
+ int key = keys.removeLast();
+
+ for (LinkedList<Integer> threadKeys : map.values()) {
+ if (threadKeys.contains(key)) {
+ found = true;
+
+ break;
+ }
+ }
+
+ assertTrue("Key was not found in any thread: " + key, found);
+ }
int min = Integer.MAX_VALUE;
int max = 0;
@@ -207,40 +208,41 @@ public class GridConcurrentLinkedHashMapMultiThreadedSelfTest extends GridCommon
public void testEvictPerSegmentOptimizedRemoves() throws Exception {
info(">>> Test grid concurrent linked hash map...");
- final int maxSize = 1000;
+ int concurLvl = 64;
+ final int maxSize = concurLvl * 30;
+ int diff = (int)(maxSize * 0.1);
ConcurrentLinkedHashMap<Integer, String> linkedMap = new ConcurrentLinkedHashMap<>(
- 32, 0.75f, 64, maxSize, PER_SEGMENT_Q_OPTIMIZED_RMV);
+ 32, 0.75f, concurLvl, maxSize, PER_SEGMENT_Q_OPTIMIZED_RMV);
int keyCnt = 1000000;
- putMultiThreaded(linkedMap, 10, keyCnt, maxSize);
-
- int diff = 10; // 1% of 1000.
+ Map<String, LinkedList<Integer>> map = putMultiThreaded(
+ linkedMap,
+ 10,
+ keyCnt,
+ maxSize * 10); // Intentionally memorize more than maxSize since in this mode LRU is not fair.
assertTrue("Invalid map size: " + linkedMap.size(), U.safeAbs(maxSize - linkedMap.size()) <= diff);
assertTrue("Invalid map sizex: " + linkedMap.sizex(), U.safeAbs(maxSize - linkedMap.sizex()) <= diff);
-// TODO IGNITE-606 - Need to fix iterators for ConcurrentLinkedHashMap in perSegment mode
-// LinkedList<Integer> keys = new LinkedList<Integer>(linkedMap.keySet());
-//
-// while (!keys.isEmpty()) {
-// boolean found = false;
-//
-// int key = keys.removeLast();
-//
-// for (LinkedList<Integer> threadKeys : map.values()) {
-// if (threadKeys.getLast() == key) {
-// threadKeys.removeLast();
-//
-// found = true;
-//
-// break;
-// }
-// }
-//
-// assertTrue("Key was not found on the top of any thread: " + key, found);
-// }
+ LinkedList<Integer> keys = new LinkedList<>(linkedMap.keySet());
+
+ while (!keys.isEmpty()) {
+ boolean found = false;
+
+ int key = keys.removeLast();
+
+ for (LinkedList<Integer> threadKeys : map.values()) {
+ if (threadKeys.contains(key)) {
+ found = true;
+
+ break;
+ }
+ }
+
+ assertTrue("Key was not found in any thread: " + key, found);
+ }
int min = Integer.MAX_VALUE;
int max = 0;
@@ -558,4 +560,4 @@ public class GridConcurrentLinkedHashMapMultiThreadedSelfTest extends GridCommon
info(">>> put get remove test complete [duration = " + (System.currentTimeMillis() - start) + ']');
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
index ead3a63..9c3389f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
@@ -19,13 +19,23 @@ package org.apache.ignite.yardstick.cache;
import java.util.Map;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
/**
* Ignite benchmark that performs transactional put operations.
*/
public class IgnitePutTxBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
/** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+ ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+ }
+
+ /** {@inheritDoc} */
@Override public boolean test(Map<Object, Object> ctx) throws Exception {
int key = nextRandom(args.range());
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
new file mode 100644
index 0000000..21275eb
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+/**
+ * Ignite benchmark that performs transactional put operations skipping key if local node is backup.
+ */
+public class IgnitePutTxPrimaryOnlyBenchmark extends IgniteCacheAbstractBenchmark {
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+ ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ int key;
+
+ Affinity<Object> aff = ignite().affinity("tx");
+ ClusterNode locNode = ignite().cluster().localNode();
+
+ for (;;) {
+ key = nextRandom(args.range());
+
+ // Exit only if primary.
+ if (aff.isPrimary(locNode, key))
+ break;
+ }
+
+ // Implicit transaction is used.
+ cache.put(key, new SampleValue(key));
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Integer, Object> cache() {
+ return ignite().cache("tx");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
new file mode 100644
index 0000000..63934e6
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+/**
+ * Ignite benchmark that performs transactional put operations skipping key if local node is backup.
+ */
+public class IgnitePutTxSkipLocalBackupBenchmark extends IgniteCacheAbstractBenchmark {
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+ ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ int key;
+
+ Affinity<Object> aff = ignite().affinity("tx");
+ ClusterNode locNode = ignite().cluster().localNode();
+
+ for (;;) {
+ key = nextRandom(args.range());
+
+ // Skip key if local node is backup.
+ if (!aff.isBackup(locNode, key))
+ break;
+ }
+
+ // Implicit transaction is used.
+ cache.put(key, new SampleValue(key));
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Integer, Object> cache() {
+ return ignite().cache("tx");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
new file mode 100644
index 0000000..83c50bd
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ *
+ */
+public class WaitMapExchangeFinishCallable implements IgniteCallable<Void> {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public Void call() throws Exception {
+ Collection<IgniteInternalCache<?, ?>> cachesx = ((IgniteKernal)ignite).cachesx(null);
+
+ for (IgniteInternalCache<?, ?> cache : cachesx) {
+ try {
+ GridDhtPartitionTopology top = cache.context().isNear() ? cache.context().near().dht().topology() :
+ cache.context().dht().topology();
+
+ BenchmarkUtils.println("Validating cache: " + cache.name());
+
+ for (;;) {
+ boolean success = true;
+
+ if (top.topologyVersion().topologyVersion() == ignite.cluster().topologyVersion()) {
+ for (Map.Entry<UUID, GridDhtPartitionMap> e : top.partitionMap(true).entrySet()) {
+ for (Map.Entry<Integer, GridDhtPartitionState> p : e.getValue().entrySet()) {
+ if (p.getValue() != GridDhtPartitionState.OWNING) {
+ BenchmarkUtils.println("Not owning partition [part=" + p.getKey() +
+ ", state=" + p.getValue() + ']');
+
+ success = false;
+
+ break;
+ }
+ }
+
+ if (!success)
+ break;
+ }
+ }
+ else {
+ BenchmarkUtils.println("Topology version is different [cache=" + top.topologyVersion() +
+ ", cluster=" + ignite.cluster().topologyVersion() + ']');
+
+ success = false;
+ }
+
+ if (!success)
+ Thread.sleep(1000);
+ else {
+ BenchmarkUtils.println("Cache state is fine: " + cache.name());
+
+ break;
+ }
+ }
+ }
+ catch (RuntimeException e1) {
+ BenchmarkUtils.println("Ignored exception: " + e1);
+ }
+ }
+
+ return null;
+ }
+}