You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vk...@apache.org on 2015/08/06 02:05:42 UTC
[1/3] incubator-ignite git commit: # ignite-1198
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-104 3d995af8a -> bc394436a
# ignite-1198
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/2173b0e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/2173b0e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/2173b0e1
Branch: refs/heads/ignite-104
Commit: 2173b0e1500d3dbc198a610042fca9b92d4f7de7
Parents: 023ffe0
Author: Alexey Goncharuk <ag...@gridgain.com>
Authored: Wed Aug 5 14:05:07 2015 -0700
Committer: Alexey Goncharuk <ag...@gridgain.com>
Committed: Wed Aug 5 14:25:13 2015 -0700
----------------------------------------------------------------------
.../org/apache/ignite/spark/IgniteContext.scala | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2173b0e1/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
index 5267244a..6e48017 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
@@ -19,7 +19,8 @@ package org.apache.ignite.spark
import org.apache.ignite.internal.IgnitionEx
-import org.apache.ignite.{Ignition, Ignite}
+import org.apache.ignite.internal.util.IgniteUtils
+import org.apache.ignite.{IgniteSystemProperties, Ignition, Ignite}
import org.apache.ignite.configuration.{CacheConfiguration, IgniteConfiguration}
import org.apache.spark.{Logging, SparkContext}
import org.apache.spark.sql.SQLContext
@@ -41,8 +42,12 @@ class IgniteContext[K, V](
private val cfgClo = new Once(cfgF)
+ private val igniteHome = IgniteUtils.getIgniteHome
+
if (!client) {
- val workers = sparkContext.getExecutorStorageStatus.length - 1
+ // Get required number of executors with default equals to number of available executors.
+ val workers = sparkContext.getConf.getInt("spark.executor.instances",
+ sparkContext.getExecutorStorageStatus.length)
if (workers <= 0)
throw new IllegalStateException("No Spark executors found to start Ignite nodes.")
@@ -125,6 +130,16 @@ class IgniteContext[K, V](
* @return Ignite instance.
*/
def ignite(): Ignite = {
+ val home = IgniteUtils.getIgniteHome
+
+ if (home == null && igniteHome != null) {
+ logInfo("Setting IGNITE_HOME from driver not as it is not available on this worker: " + igniteHome)
+
+ IgniteUtils.nullifyHomeDirectory()
+
+ System.setProperty(IgniteSystemProperties.IGNITE_HOME, igniteHome)
+ }
+
val igniteCfg = cfgClo()
try {
[3/3] incubator-ignite git commit: IGNITE-104 - Ordered ATOMIC updates
Posted by vk...@apache.org.
IGNITE-104 - Ordered ATOMIC updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/bc394436
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/bc394436
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/bc394436
Branch: refs/heads/ignite-104
Commit: bc394436a89c945cbca1c4f158b00f19951cf47e
Parents: 6c7358d
Author: Valentin Kulichenko <vk...@gridgain.com>
Authored: Wed Aug 5 17:05:35 2015 -0700
Committer: Valentin Kulichenko <vk...@gridgain.com>
Committed: Wed Aug 5 17:05:35 2015 -0700
----------------------------------------------------------------------
.../dht/atomic/GridAtomicRequestTopic.java | 44 ++++++++++++++++----
.../dht/atomic/GridDhtAtomicCache.java | 8 ++--
.../dht/atomic/GridDhtAtomicUpdateFuture.java | 2 +-
.../dht/atomic/GridNearAtomicUpdateFuture.java | 2 +-
4 files changed, 42 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/bc394436/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridAtomicRequestTopic.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridAtomicRequestTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridAtomicRequestTopic.java
index 9feb409..709f739 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridAtomicRequestTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridAtomicRequestTopic.java
@@ -28,13 +28,41 @@ class GridAtomicRequestTopic implements Externalizable {
private static final long serialVersionUID = 0L;
/** */
+ private static final byte NEAR_UPDATE_REQ = 1;
+
+ /** */
+ private static final byte DHT_UPDATE_REQ = 2;
+
+ /** */
private int cacheId;
/** */
private int part;
/** */
- private boolean near;
+ private byte type;
+
+ /**
+ * Near request topic.
+ *
+ * @param cacheId Cache ID.
+ * @param part Partition.
+ * @return Topic.
+ */
+ static GridAtomicRequestTopic nearUpdateRequest(int cacheId, int part) {
+ return new GridAtomicRequestTopic(cacheId, part, NEAR_UPDATE_REQ);
+ }
+
+ /**
+ * DHT request topic.
+ *
+ * @param cacheId Cache ID.
+ * @param part Partition.
+ * @return Topic.
+ */
+ static GridAtomicRequestTopic dhtUpdateRequest(int cacheId, int part) {
+ return new GridAtomicRequestTopic(cacheId, part, DHT_UPDATE_REQ);
+ }
/**
* For {@link Externalizable}.
@@ -46,12 +74,12 @@ class GridAtomicRequestTopic implements Externalizable {
/**
* @param cacheId Cache ID.
* @param part Partition.
- * @param near Near flag.
+ * @param type Type.
*/
- GridAtomicRequestTopic(int cacheId, int part, boolean near) {
+ private GridAtomicRequestTopic(int cacheId, int part, byte type) {
this.cacheId = cacheId;
this.part = part;
- this.near = near;
+ this.type = type;
}
/** {@inheritDoc} */
@@ -62,7 +90,7 @@ class GridAtomicRequestTopic implements Externalizable {
GridAtomicRequestTopic topic = (GridAtomicRequestTopic)o;
- return cacheId == topic.cacheId && part == topic.part && near == topic.near;
+ return cacheId == topic.cacheId && part == topic.part && type == topic.type;
}
/** {@inheritDoc} */
@@ -70,7 +98,7 @@ class GridAtomicRequestTopic implements Externalizable {
int res = cacheId;
res = 31 * res + part;
- res = 31 * res + (near ? 1 : 0);
+ res = 31 * res + type;
return res;
}
@@ -79,14 +107,14 @@ class GridAtomicRequestTopic implements Externalizable {
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeInt(cacheId);
out.writeInt(part);
- out.writeBoolean(near);
+ out.writeByte(type);
}
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
cacheId = in.readInt();
part = in.readInt();
- near = in.readBoolean();
+ type = in.readByte();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/bc394436/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 01694d7..6949ae2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -188,7 +188,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
});
for (int part = 0; part < ctx.affinity().partitions(); part++) {
- Object nearTopic = new GridAtomicRequestTopic(ctx.cacheId(), part, true);
+ Object nearTopic = GridAtomicRequestTopic.nearUpdateRequest(ctx.cacheId(), part);
ctx.io().addPerTopicHandler(nearTopic, new CI2<UUID, GridNearAtomicUpdateRequest>() {
@Override public void apply(UUID nodeId, GridNearAtomicUpdateRequest req) {
@@ -196,7 +196,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
});
- Object dhtTopic = new GridAtomicRequestTopic(ctx.cacheId(), part, false);
+ Object dhtTopic = GridAtomicRequestTopic.dhtUpdateRequest(ctx.cacheId(), part);
ctx.io().addPerTopicHandler(dhtTopic, new CI2<UUID, GridDhtAtomicUpdateRequest>() {
@Override public void apply(UUID nodeId, GridDhtAtomicUpdateRequest req) {
@@ -239,8 +239,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
buf.finish();
for (int part = 0; part < ctx.affinity().partitions(); part++) {
- ctx.io().removePerTopicHandler(new GridAtomicRequestTopic(ctx.cacheId(), part, true));
- ctx.io().removePerTopicHandler(new GridAtomicRequestTopic(ctx.cacheId(), part, false));
+ ctx.io().removePerTopicHandler(GridAtomicRequestTopic.nearUpdateRequest(ctx.cacheId(), part));
+ ctx.io().removePerTopicHandler(GridAtomicRequestTopic.dhtUpdateRequest(ctx.cacheId(), part));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/bc394436/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
index 7823a52..63edcaa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
@@ -352,7 +352,7 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
log.debug("Sending DHT atomic update request [nodeId=" + nodeId + ", req=" + req + ']');
if (part >= 0) {
- Object topic = new GridAtomicRequestTopic(cctx.cacheId(), part, false);
+ Object topic = GridAtomicRequestTopic.dhtUpdateRequest(cctx.cacheId(), part);
cctx.io().sendSequentialMessage(nodeId, topic, req, cctx.ioPolicy());
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/bc394436/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index 5b364a5..66f0300 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -1091,7 +1091,7 @@ public class GridNearAtomicUpdateFuture extends GridFutureAdapter<Object>
private void sendRequest(GridAtomicMappingKey mappingKey, GridNearAtomicUpdateRequest req)
throws IgniteCheckedException {
if (mappingKey.partition() >= 0) {
- Object topic = new GridAtomicRequestTopic(cctx.cacheId(), mappingKey.partition(), true);
+ Object topic = GridAtomicRequestTopic.nearUpdateRequest(cctx.cacheId(), mappingKey.partition());
cctx.io().sendSequentialMessage(mappingKey.nodeId(), topic, req, cctx.ioPolicy());
}
[2/3] incubator-ignite git commit: Merge branches 'ignite-104' and
'master' of https://git-wip-us.apache.org/repos/asf/incubator-ignite into
ignite-104
Posted by vk...@apache.org.
Merge branches 'ignite-104' and 'master' of https://git-wip-us.apache.org/repos/asf/incubator-ignite into ignite-104
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/6c7358db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/6c7358db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/6c7358db
Branch: refs/heads/ignite-104
Commit: 6c7358db22e008f383acb356a3922107531c063b
Parents: 3d995af 2173b0e
Author: Valentin Kulichenko <vk...@gridgain.com>
Authored: Wed Aug 5 16:54:04 2015 -0700
Committer: Valentin Kulichenko <vk...@gridgain.com>
Committed: Wed Aug 5 16:54:04 2015 -0700
----------------------------------------------------------------------
.../org/apache/ignite/spark/IgniteContext.scala | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------