You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/06/23 16:22:56 UTC
[1/6] incubator-ignite git commit: ignite-1048 create/lock entries
before topology read lock
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-917 9946b2aed -> 0b0b9dcaa
ignite-1048 create/lock entries before topology read lock
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/be881a36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/be881a36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/be881a36
Branch: refs/heads/ignite-917
Commit: be881a367ef02a724c2b84aba8f72c6effadc2b2
Parents: 1c66078
Author: sboikov <sb...@gridgain.com>
Authored: Tue Jun 23 10:52:47 2015 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Jun 23 10:52:47 2015 +0300
----------------------------------------------------------------------
.../cache/distributed/dht/GridDhtLocalPartition.java | 3 ++-
.../cache/distributed/dht/atomic/GridDhtAtomicCache.java | 9 ++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/be881a36/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 3a577a7..87c7f0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -236,7 +236,8 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
void onAdded(GridDhtCacheEntry entry) {
GridDhtPartitionState state = state();
- assert state != EVICTED : "Adding entry to invalid partition: " + this;
+ if (state == EVICTED)
+ throw new GridDhtInvalidPartitionException(id, "Adding entry to invalid partition [part=" + id + ']');
map.put(entry.key(), entry);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/be881a36/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 8630421..7bec302 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1028,7 +1028,10 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
IgniteCacheExpiryPolicy expiry = null;
try {
- List<GridDhtCacheEntry> locked = null;
+ // If batch store update is enabled, we need to lock all entries.
+ // First, need to acquire locks on cache entries, then check filter.
+ List<GridDhtCacheEntry> locked = lockEntries(keys, req.topologyVersion());
+
Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
try {
@@ -1058,10 +1061,6 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
return;
}
- // If batch store update is enabled, we need to lock all entries.
- // First, need to acquire locks on cache entries, then check filter.
- locked = lockEntries(keys, req.topologyVersion());
-
boolean hasNear = ctx.discovery().cacheNearNode(node, name());
GridCacheVersion ver = req.updateVersion();
[2/6] incubator-ignite git commit: Merge remote-tracking branch
'remotes/origin/ignite-sprint-7' into ignite-1048
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/ignite-sprint-7' into ignite-1048
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/a2d2c9b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/a2d2c9b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/a2d2c9b9
Branch: refs/heads/ignite-917
Commit: a2d2c9b96e9bd86f2f711df5da835c95af57b612
Parents: be881a3 c7ba154
Author: sboikov <sb...@gridgain.com>
Authored: Tue Jun 23 12:56:44 2015 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Jun 23 12:56:44 2015 +0300
----------------------------------------------------------------------
.../replicated/GridCacheReplicatedFailoverSelfTest.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
[4/6] incubator-ignite git commit: Merge remote-tracking branch
'remotes/origin/ignite-1048' into ignite-sprint-7
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/ignite-1048' into ignite-sprint-7
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/dfe95bdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/dfe95bdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/dfe95bdf
Branch: refs/heads/ignite-917
Commit: dfe95bdfb98663134809ca456f63e100fb1e7c26
Parents: 2bb6e0f a2d2c9b
Author: sboikov <sb...@gridgain.com>
Authored: Tue Jun 23 16:38:38 2015 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Jun 23 16:38:38 2015 +0300
----------------------------------------------------------------------
.../cache/distributed/dht/GridDhtLocalPartition.java | 3 ++-
.../cache/distributed/dht/atomic/GridDhtAtomicCache.java | 9 ++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[6/6] incubator-ignite git commit: Merge branch 'ignite-sprint-7'
into ignite-917
Posted by sb...@apache.org.
Merge branch 'ignite-sprint-7' into ignite-917
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/0b0b9dca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/0b0b9dca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/0b0b9dca
Branch: refs/heads/ignite-917
Commit: 0b0b9dcaa56edfbfd52da8cb2929512f6a1c96cd
Parents: 9946b2a 359b431
Author: ashutak <as...@gridgain.com>
Authored: Tue Jun 23 17:23:15 2015 +0300
Committer: ashutak <as...@gridgain.com>
Committed: Tue Jun 23 17:23:15 2015 +0300
----------------------------------------------------------------------
.../configuration/IgniteReflectionFactory.java | 81 ++++++-
.../distributed/dht/GridDhtLocalPartition.java | 3 +-
.../dht/atomic/GridDhtAtomicCache.java | 9 +-
.../hadoop/fs/HadoopFileSystemCacheUtils.java | 241 +++++++++++++++++++
4 files changed, 323 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[3/6] incubator-ignite git commit: [IGNITE-980]: added missing file
Posted by sb...@apache.org.
[IGNITE-980]: added missing file
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/2bb6e0f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/2bb6e0f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/2bb6e0f8
Branch: refs/heads/ignite-917
Commit: 2bb6e0f887c99de70e479c48992f5a0a86cbf5ec
Parents: 3b49184
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Jun 23 15:02:23 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Jun 23 15:02:23 2015 +0300
----------------------------------------------------------------------
.../hadoop/fs/HadoopFileSystemCacheUtils.java | 241 +++++++++++++++++++
1 file changed, 241 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2bb6e0f8/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemCacheUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemCacheUtils.java
new file mode 100644
index 0000000..f94b932
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopFileSystemCacheUtils.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.mapreduce.*;
+import org.apache.ignite.*;
+import org.apache.ignite.hadoop.fs.v1.*;
+import org.apache.ignite.internal.util.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+
+/**
+ * File system cache utility methods used by Map-Reduce tasks and jobs.
+ */
+public class HadoopFileSystemCacheUtils {
+ /**
+ * A common static factory method. Creates new HadoopLazyConcurrentMap.
+ * @return a new HadoopLazyConcurrentMap.
+ */
+ public static HadoopLazyConcurrentMap<FsCacheKey, FileSystem> createHadoopLazyConcurrentMap() {
+ return new HadoopLazyConcurrentMap<>(
+ new HadoopLazyConcurrentMap.ValueFactory<FsCacheKey, FileSystem>() {
+ @Override public FileSystem createValue(FsCacheKey key) {
+ try {
+ assert key != null;
+
+ // Explicitly disable FileSystem caching:
+ URI uri = key.uri();
+
+ String scheme = uri.getScheme();
+
+ // Copy the configuration to avoid altering the external object.
+ Configuration cfg = new Configuration(key.configuration());
+
+ String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(scheme);
+
+ cfg.setBoolean(prop, true);
+
+ return FileSystem.get(uri, cfg, key.user());
+ }
+ catch (IOException | InterruptedException ioe) {
+ throw new IgniteException(ioe);
+ }
+ }
+ }
+ );
+ }
+
+ /**
+ * Gets non-null user name as per the Hadoop viewpoint.
+ * @param cfg the Hadoop job configuration, may be null.
+ * @return the user name, never null.
+ */
+ private static String getMrHadoopUser(Configuration cfg) throws IOException {
+ String user = cfg.get(MRJobConfig.USER_NAME);
+
+ if (user == null)
+ user = IgniteHadoopFileSystem.getFsHadoopUser();
+
+ return user;
+ }
+
+ /**
+ * Common method to get the V1 file system in MapRed engine.
+ * It gets the filesystem for the user specified in the
+ * configuration with {@link MRJobConfig#USER_NAME} property.
+ * The file systems are created and cached in the given map upon first request.
+ *
+ * @param uri The file system uri.
+ * @param cfg The configuration.
+ * @param map The caching map.
+ * @return The file system.
+ * @throws IOException On error.
+ */
+ public static FileSystem fileSystemForMrUserWithCaching(@Nullable URI uri, Configuration cfg,
+ HadoopLazyConcurrentMap<FsCacheKey, FileSystem> map)
+ throws IOException {
+ assert map != null;
+ assert cfg != null;
+
+ final String usr = getMrHadoopUser(cfg);
+
+ assert usr != null;
+
+ if (uri == null)
+ uri = FileSystem.getDefaultUri(cfg);
+
+ final FileSystem fs;
+
+ try {
+ final FsCacheKey key = new FsCacheKey(uri, usr, cfg);
+
+ fs = map.getOrCreate(key);
+ }
+ catch (IgniteException ie) {
+ throw new IOException(ie);
+ }
+
+ assert fs != null;
+ assert !(fs instanceof IgniteHadoopFileSystem) || F.eq(usr, ((IgniteHadoopFileSystem)fs).user());
+
+ return fs;
+ }
+
+ /**
+ * Takes Fs URI using logic similar to that used in FileSystem#get(1,2,3).
+ * @param uri0 The uri.
+ * @param cfg The cfg.
+ * @return Correct URI.
+ */
+ private static URI fixUri(URI uri0, Configuration cfg) {
+ if (uri0 == null)
+ return FileSystem.getDefaultUri(cfg);
+
+ String scheme = uri0.getScheme();
+ String authority = uri0.getAuthority();
+
+ if (authority == null) {
+ URI dfltUri = FileSystem.getDefaultUri(cfg);
+
+ if (scheme == null || (scheme.equals(dfltUri.getScheme()) && dfltUri.getAuthority() != null))
+ return dfltUri;
+ }
+
+ return uri0;
+ }
+
+ /**
+ * Note that configuration is not a part of the key.
+ * It is used solely to initialize the first instance
+ * that is created for the key.
+ */
+ public static final class FsCacheKey {
+ /** */
+ private final URI uri;
+
+ /** */
+ private final String usr;
+
+ /** */
+ private final String equalityKey;
+
+ /** */
+ private final Configuration cfg;
+
+ /**
+ * Constructor
+ */
+ public FsCacheKey(URI uri, String usr, Configuration cfg) {
+ assert uri != null;
+ assert usr != null;
+ assert cfg != null;
+
+ this.uri = fixUri(uri, cfg);
+ this.usr = usr;
+ this.cfg = cfg;
+
+ this.equalityKey = createEqualityKey();
+ }
+
+ /**
+ * Creates String key used for equality and hashing.
+ */
+ private String createEqualityKey() {
+ GridStringBuilder sb = new GridStringBuilder("(").a(usr).a(")@");
+
+ if (uri.getScheme() != null)
+ sb.a(uri.getScheme().toLowerCase());
+
+ sb.a("://");
+
+ if (uri.getAuthority() != null)
+ sb.a(uri.getAuthority().toLowerCase());
+
+ return sb.toString();
+ }
+
+ /**
+ * The URI.
+ */
+ public URI uri() {
+ return uri;
+ }
+
+ /**
+ * The User.
+ */
+ public String user() {
+ return usr;
+ }
+
+ /**
+ * The Configuration.
+ */
+ public Configuration configuration() {
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("SimplifiableIfStatement")
+ @Override public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj == null || getClass() != obj.getClass())
+ return false;
+
+ return equalityKey.equals(((FsCacheKey)obj).equalityKey);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return equalityKey.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return equalityKey;
+ }
+ }
+}
[5/6] incubator-ignite git commit: # ignite-sprint-7:
IgniteReflectionFactory javadoc
Posted by sb...@apache.org.
# ignite-sprint-7: IgniteReflectionFactory javadoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/359b4314
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/359b4314
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/359b4314
Branch: refs/heads/ignite-917
Commit: 359b43144046bc683f9e28794eb7e7bc9c63ebc1
Parents: dfe95bd
Author: ashutak <as...@gridgain.com>
Authored: Tue Jun 23 16:59:44 2015 +0300
Committer: ashutak <as...@gridgain.com>
Committed: Tue Jun 23 16:59:44 2015 +0300
----------------------------------------------------------------------
.../configuration/IgniteReflectionFactory.java | 81 ++++++++++++++++++--
1 file changed, 76 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/359b4314/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
index 3222938..af73737 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
@@ -26,26 +26,91 @@ import java.lang.reflect.*;
import java.util.*;
/**
- * Convenience class for reflection-based object creation.
+ * Factory implementation that use reflection to create instance of given class.
+ * <p>
+ * There are 2 modes of factory: singleton and non-sigletton.
+ * <p>
+ * Class that should be created by {@link IgniteReflectionFactory} (component class) have to be
+ * public java POJO with public setters for field
+ * for which property injection will be used (see {@link #setProperties(Map)}).
+ * <p>
+ * <h1 class="header">Configuration</h1>
+ * <h2 class="header">Mandatory</h2>
+ * The following configuration parameters are mandatory:
+ * <ul>
+ * <li>Component class - class to be created (see {@link #setComponentClass(Class)}.
+ * It have to be public java POJO class with default constructor
+ * and public setters to be used by properties injection (see {@link #setProperties(Map)})</li>
+ * </ul>
+ * <h2 class="header">Optional</h2>
+ * The following configuration parameters are optional:
+ * <ul>
+ * </li>
+ * <li>Singleton mode (see {@link #setSingleton(boolean)})</li>
+ * <li>Properties map (see {@link #setProperties(Map)}</li>
+ * <li>With method (see {@link #setWithMethod(Object, String, Serializable)}</li>
+ * </ul>
+ * <h2 class="header">Java Example</h2>
+ * <pre name="code" class="java">
+ * Factory<CacheStoreSessionListener> factory =
+ * new IgniteReflectionFactory<CacheStoreSessionListener>(MyCacheStoreSessionListener.class);
+ *
+ * CacheConfiguration cc = new CacheConfiguration()
+ * .setCacheStoreSessionListenerFactories(factory);
+ *
+ * IgniteConfiguration cfg = new IgniteConfiguration()
+ * .setCacheConfiguration(cc);
+ *
+ * // Start grid.
+ * Ignition.start(cfg);
+ * </pre>
+ * <h2 class="header">Spring Example</h2>
+ * TcpDiscoverySpi can be configured from Spring XML configuration file:
+ * <pre name="code" class="xml">
+ * <bean id="grid.custom.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+ * ...
+ * <property name="cacheConfiguration">
+ * <list>
+ * <bean class="org.apache.ignite.configuration.CacheConfiguration">
+ * ...
+ * <property name="cacheStoreSessionListenerFactories">
+ * <list>
+ * <bean class="org.apache.ignite.configuration.IgniteReflectionFactory">
+ * <property name="componentClass" value="custom.project.MyCacheStoreSessionListener"/>
+ * </bean>
+ * </list>
+ * </property>
+ * ...
+ * </bean>
+ * </list>
+ * </property>
+ * ...
+ * </bean>
+ * </pre>
+ * <p>
+ * <img src="http://ignite.incubator.apache.org/images/spring-small.png">
+ * <br>
+ * For information about Spring framework visit <a href="http://www.springframework.org/">www.springframework.org</a>
+ * @see Factory
*/
public class IgniteReflectionFactory<T> implements Factory<T> {
/** */
private static final long serialVersionUID = 0L;
- /** */
+ /** Singletom mode */
private volatile boolean singleton;
- /** */
+ /** Component class */
private volatile Class<? extends T> cls;
- /** */
+ /** Properties */
private volatile Map<String, Serializable> props;
/** */
private transient T instance;
/**
- *
+ * Default constructor.
*/
public IgniteReflectionFactory() {
// No-op.
@@ -98,6 +163,9 @@ public class IgniteReflectionFactory<T> implements Factory<T> {
}
/**
+ * Gets a map of properties. Map contains entries of component class field name
+ * to value of the filed which will be used as initial value.
+ *
* @return Properties.
*/
public Map<String, Serializable> getProperties() {
@@ -105,6 +173,9 @@ public class IgniteReflectionFactory<T> implements Factory<T> {
}
/**
+ * Sets a map of properties. Map contains entries of component class field name
+ * to a value of the filed which will be used as initial value.
+ *
* @param props Properties.
*/
public void setProperties(Map<String, Serializable> props) {