You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dm...@apache.org on 2016/06/02 13:26:06 UTC
[1/8] ignite git commit: IGNITE-3200: Resolved classloader leak.
Repository: ignite
Updated Branches:
refs/heads/master 3c3ed0563 -> 4cc10982e
IGNITE-3200: Resolved classloader leak.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/56c60562
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/56c60562
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/56c60562
Branch: refs/heads/master
Commit: 56c605622dbecf71f3ee87cc41a16d2ee87e92d3
Parents: a28e16e
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu May 26 11:15:48 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue May 31 11:28:18 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/internal/util/ClassCache.java | 32 ++++++++++++++++++++
.../ignite/internal/util/IgniteUtils.java | 6 +++-
.../processors/hadoop/HadoopClassLoader.java | 23 +++++++++++++-
3 files changed, 59 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/56c60562/modules/core/src/main/java/org/apache/ignite/internal/util/ClassCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/ClassCache.java b/modules/core/src/main/java/org/apache/ignite/internal/util/ClassCache.java
new file mode 100644
index 0000000..7d6edbe
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/ClassCache.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+/**
+ * Class cache.
+ */
+public interface ClassCache {
+ /**
+ * Get class for the given name.
+ *
+ * @param clsName Class name.
+ * @return Class.
+ * @throws ClassNotFoundException If not found.
+ */
+ public Class<?> getFromCache(String clsName) throws ClassNotFoundException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/56c60562/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index ed8e100..cbefd7d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -8232,7 +8232,11 @@ public abstract class IgniteUtils {
if (cls != null)
return cls;
- if (ldr == null)
+ if (ldr != null) {
+ if (ldr instanceof ClassCache)
+ return ((ClassCache)ldr).getFromCache(clsName);
+ }
+ else
ldr = gridClassLoader;
ConcurrentMap<String, Class> ldrMap = classCache.get(ldr);
http://git-wip-us.apache.org/repos/asf/ignite/blob/56c60562/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
index 270b31d..4448b2d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -31,10 +31,14 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.hadoop.v2.HadoopDaemon;
import org.apache.ignite.internal.processors.hadoop.v2.HadoopShutdownHookManager;
+import org.apache.ignite.internal.util.ClassCache;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -59,7 +63,7 @@ import org.objectweb.asm.commons.RemappingClassAdapter;
* Also supports class parsing for finding dependencies which contain transitive dependencies
* unavailable for parent.
*/
-public class HadoopClassLoader extends URLClassLoader {
+public class HadoopClassLoader extends URLClassLoader implements ClassCache {
/**
* We are very parallel capable.
*/
@@ -88,6 +92,9 @@ public class HadoopClassLoader extends URLClassLoader {
/** */
private static final Map<String, byte[]> bytesCache = new ConcurrentHashMap8<>();
+ /** Class cache. */
+ private final ConcurrentMap<String, Class> cacheMap = new ConcurrentHashMap<>();
+
/** Diagnostic name of this class loader. */
@SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
private final String name;
@@ -282,6 +289,20 @@ public class HadoopClassLoader extends URLClassLoader {
}
}
+ /** {@inheritDoc} */
+ @Override public Class<?> getFromCache(String clsName) throws ClassNotFoundException {
+ Class<?> cls = cacheMap.get(clsName);
+
+ if (cls == null) {
+ Class old = cacheMap.putIfAbsent(clsName, cls = Class.forName(clsName, true, this));
+
+ if (old != null)
+ cls = old;
+ }
+
+ return cls;
+ }
+
/**
* @param name Class name.
* @param resolve Resolve class.
[7/8] ignite git commit: IGNITE-2655: AffinityFunction: primary and
backup copies in different locations Reviewed and merged by Denis Magda
(dmagda@gridgain.com)
Posted by dm...@apache.org.
IGNITE-2655: AffinityFunction: primary and backup copies in different locations
Reviewed and merged by Denis Magda (dmagda@gridgain.com)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f175d3c6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f175d3c6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f175d3c6
Branch: refs/heads/master
Commit: f175d3c670025bd619ec347dba2a5c5f68f4cc32
Parents: 3ca9feb
Author: Vladislav Pyatkov <vl...@gmail.com>
Authored: Thu Jun 2 16:14:10 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu Jun 2 16:14:10 2016 +0300
----------------------------------------------------------------------
.../affinity/fair/FairAffinityFunction.java | 81 +++++++++++-
.../rendezvous/RendezvousAffinityFunction.java | 39 +++++-
...ityFunctionBackupFilterAbstractSelfTest.java | 131 ++++++++++++++++++-
...airAffinityFunctionBackupFilterSelfTest.java | 9 ++
...ousAffinityFunctionBackupFilterSelfTest.java | 9 ++
5 files changed, 260 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f175d3c6/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
index b42b683..b6b14ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunction.java
@@ -102,6 +102,9 @@ public class FairAffinityFunction implements AffinityFunction {
/** Optional backup filter. First node is primary, second node is a node being tested. */
private IgniteBiPredicate<ClusterNode, ClusterNode> backupFilter;
+ /** Optional affinity backups filter. The first node is a node being tested, the second is a list of nodes that are already assigned for a given partition (primary node is the first in the list). */
+ private IgniteBiPredicate<ClusterNode, List<ClusterNode>> affinityBackupFilter;
+
/**
* Empty constructor with all defaults.
*/
@@ -220,12 +223,40 @@ public class FairAffinityFunction implements AffinityFunction {
* Note that {@code backupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
*
* @param backupFilter Optional backup filter.
+ * @deprecated Use {@code affinityBackupFilter} instead.
*/
+ @Deprecated
public void setBackupFilter(@Nullable IgniteBiPredicate<ClusterNode, ClusterNode> backupFilter) {
this.backupFilter = backupFilter;
}
/**
+ * Gets optional backup filter. If not {@code null}, backups will be selected
+ * from all nodes that pass this filter. First node passed to this filter is a node being tested,
+ * and the second parameter is a list of nodes that are already assigned for a given partition (primary node is the first in the list).
+ * <p>
+ * Note that {@code affinityBackupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
+ *
+ * @return Optional backup filter.
+ */
+ @Nullable public IgniteBiPredicate<ClusterNode, List<ClusterNode>> getAffinityBackupFilter() {
+ return affinityBackupFilter;
+ }
+
+ /**
+ * Sets optional backup filter. If provided, then backups will be selected from all
+ * nodes that pass this filter. First node being passed to this filter is a node being tested,
+ * and the second parameter is a list of nodes that are already assigned for a given partition (primary node is the first in the list).
+ * <p>
+ * Note that {@code affinityBackupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
+ *
+ * @param affinityBackupFilter Optional backup filter.
+ */
+ public void setAffinityBackupFilter(@Nullable IgniteBiPredicate<ClusterNode, List<ClusterNode>> affinityBackupFilter) {
+ this.affinityBackupFilter = affinityBackupFilter;
+ }
+
+ /**
* Checks flag to exclude same-host-neighbors from being backups of each other (default is {@code false}).
* <p>
* Note that {@code backupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
@@ -865,6 +896,7 @@ public class FairAffinityFunction implements AffinityFunction {
* @param tier Tier.
* @param node Node.
* @param allowNeighbors Allow neighbors.
+ * @return {@code true} if the partition is assignable to the node.
*/
private boolean isAssignable(int part, int tier, final ClusterNode node, boolean allowNeighbors) {
if (containsPartition(part, node))
@@ -872,9 +904,50 @@ public class FairAffinityFunction implements AffinityFunction {
if (exclNeighbors)
return allowNeighbors || !neighborsContainPartition(node, part);
- else if (backupFilter == null)
- return true;
- else {
+ else if (affinityBackupFilter != null) {
+ List<ClusterNode> assigment = assignments.get(part);
+
+ assert assigment.size() > 0;
+
+ List<ClusterNode> newAssignment;
+
+ if (tier == 0) {
+ for (int t = 1; t < assigment.size(); t++) {
+ newAssignment = new ArrayList<>(assigment.size() - 1);
+
+ newAssignment.add(node);
+
+ if (t != 1)
+ newAssignment.addAll(assigment.subList(1, t));
+
+ if (t + 1 < assigment.size())
+ newAssignment.addAll(assigment.subList(t + 1, assigment.size()));
+
+ if (!affinityBackupFilter.apply(assigment.get(t), newAssignment))
+ return false;
+
+ }
+
+ return true;
+ }
+ else if (tier < assigment.size()) {
+ newAssignment = new ArrayList<>(assigment.size() - 1);
+
+ int i = 0;
+
+ for (ClusterNode assignmentNode: assigment) {
+ if (i != tier)
+ newAssignment.add(assignmentNode);
+
+ i++;
+ }
+ }
+ else
+ newAssignment = assigment;
+
+ return affinityBackupFilter.apply(node, newAssignment);
+ }
+ else if (backupFilter != null) {
if (tier == 0) {
List<ClusterNode> assigment = assignments.get(part);
@@ -891,6 +964,8 @@ public class FairAffinityFunction implements AffinityFunction {
else
return (backupFilter.apply(assignments.get(part).get(0), node));
}
+ else
+ return true;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/f175d3c6/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
index 37258d4..990eba1 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
@@ -111,6 +111,11 @@ public class RendezvousAffinityFunction implements AffinityFunction, Externaliza
/** Optional backup filter. First node is primary, second node is a node being tested. */
private IgniteBiPredicate<ClusterNode, ClusterNode> backupFilter;
+ /** Optional affinity backups filter. The first node is a node being tested,
+ * the second is a list of nodes that are already assigned for a given partition (the first node in the list
+ * is primary). */
+ private IgniteBiPredicate<ClusterNode, List<ClusterNode>> affinityBackupFilter;
+
/** Hash ID resolver. */
private AffinityNodeHashResolver hashIdRslvr = null;
@@ -277,12 +282,40 @@ public class RendezvousAffinityFunction implements AffinityFunction, Externaliza
* Note that {@code backupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
*
* @param backupFilter Optional backup filter.
+ * @deprecated Use {@code affinityBackupFilter} instead.
*/
+ @Deprecated
public void setBackupFilter(@Nullable IgniteBiPredicate<ClusterNode, ClusterNode> backupFilter) {
this.backupFilter = backupFilter;
}
/**
+ * Gets optional backup filter. If not {@code null}, backups will be selected
+ * from all nodes that pass this filter. First node passed to this filter is a node being tested,
+ * and the second parameter is a list of nodes that are already assigned for a given partition (primary node is the first in the list).
+ * <p>
+ * Note that {@code affinityBackupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
+ *
+ * @return Optional backup filter.
+ */
+ @Nullable public IgniteBiPredicate<ClusterNode, List<ClusterNode>> getAffinityBackupFilter() {
+ return affinityBackupFilter;
+ }
+
+ /**
+ * Sets optional backup filter. If provided, then backups will be selected from all
+ * nodes that pass this filter. First node being passed to this filter is a node being tested,
+ * and the second parameter is a list of nodes that are already assigned for a given partition (primary node is the first in the list).
+ * <p>
+ * Note that {@code affinityBackupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
+ *
+ * @param affinityBackupFilter Optional backup filter.
+ */
+ public void setAffinityBackupFilter(@Nullable IgniteBiPredicate<ClusterNode, List<ClusterNode>> affinityBackupFilter) {
+ this.affinityBackupFilter = affinityBackupFilter;
+ }
+
+ /**
* Checks flag to exclude same-host-neighbors from being backups of each other (default is {@code false}).
* <p>
* Note that {@code backupFilter} is ignored if {@code excludeNeighbors} is set to {@code true}.
@@ -384,7 +417,11 @@ public class RendezvousAffinityFunction implements AffinityFunction, Externaliza
if (!allNeighbors.contains(node))
res.add(node);
}
- else if (backupFilter == null || backupFilter.apply(primary, node))
+ else if (affinityBackupFilter != null && affinityBackupFilter.apply(node, res))
+ res.add(next.get2());
+ else if (backupFilter != null && backupFilter.apply(primary, node))
+ res.add(next.get2());
+ else if (affinityBackupFilter == null && backupFilter == null)
res.add(next.get2());
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f175d3c6/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionBackupFilterAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionBackupFilterAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionBackupFilterAbstractSelfTest.java
index 3bf41c1..f01f5d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionBackupFilterAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionBackupFilterAbstractSelfTest.java
@@ -18,6 +18,10 @@
package org.apache.ignite.cache.affinity;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cluster.ClusterNode;
@@ -29,6 +33,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -41,15 +46,18 @@ public abstract class AffinityFunctionBackupFilterAbstractSelfTest extends GridC
/** Ip finder. */
private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
- /** Backup count. */
- private static final int BACKUPS = 1;
-
/** Split attribute name. */
private static final String SPLIT_ATTRIBUTE_NAME = "split-attribute";
/** Split attribute value. */
private String splitAttrVal;
+ /** Attribute value for first node group. */
+ public static final String FIRST_NODE_GROUP = "A";
+
+ /** Backup count. */
+ private int backups = 1;
+
/** Test backup filter. */
protected static final IgniteBiPredicate<ClusterNode, ClusterNode> backupFilter =
new IgniteBiPredicate<ClusterNode, ClusterNode>() {
@@ -61,13 +69,63 @@ public abstract class AffinityFunctionBackupFilterAbstractSelfTest extends GridC
}
};
+ /** Test backup filter. */
+ protected static final IgniteBiPredicate<ClusterNode, List<ClusterNode>> affinityBackupFilter =
+ new IgniteBiPredicate<ClusterNode, List<ClusterNode>>() {
+ @Override public boolean apply(ClusterNode node, List<ClusterNode> assigned) {
+ assert node != null : "primary is null";
+ assert assigned != null : "backup is null";
+
+ Map<String, Integer> backupAssignedAttribute = getAttributeStatistic(assigned);
+
+ String nodeAttributeValue = node.attribute(SPLIT_ATTRIBUTE_NAME);
+
+ if (FIRST_NODE_GROUP.equals(nodeAttributeValue)
+ && backupAssignedAttribute.get(FIRST_NODE_GROUP) < 2)
+ return true;
+
+ return backupAssignedAttribute.get(nodeAttributeValue).equals(0);
+ }
+ };
+
+ /**
+ * @param nodes List of cluster nodes.
+ * @return Statistic.
+ */
+ @NotNull private static Map<String, Integer> getAttributeStatistic(Collection<ClusterNode> nodes) {
+ Map<String, Integer> backupAssignedAttribute = new HashMap<>();
+
+ backupAssignedAttribute.put(FIRST_NODE_GROUP, 0);
+
+ backupAssignedAttribute.put("B", 0);
+
+ backupAssignedAttribute.put("C", 0);
+
+ for (ClusterNode assignedNode: nodes) {
+ if (assignedNode == null)
+ continue;
+
+ String val = assignedNode.attribute(SPLIT_ATTRIBUTE_NAME);
+
+ Integer count = backupAssignedAttribute.get(val);
+
+ backupAssignedAttribute.put(val, count + 1);
+ }
+ return backupAssignedAttribute;
+ }
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
CacheConfiguration cacheCfg = defaultCacheConfiguration();
cacheCfg.setCacheMode(PARTITIONED);
- cacheCfg.setBackups(BACKUPS);
- cacheCfg.setAffinity(affinityFunction());
+ cacheCfg.setBackups(backups);
+
+ if (backups < 2)
+ cacheCfg.setAffinity(affinityFunction());
+ else
+ cacheCfg.setAffinity(affinityFunctionWithAffinityBackupFilter());
+
cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
cacheCfg.setRebalanceMode(SYNC);
cacheCfg.setAtomicityMode(TRANSACTIONAL);
@@ -90,9 +148,15 @@ public abstract class AffinityFunctionBackupFilterAbstractSelfTest extends GridC
protected abstract AffinityFunction affinityFunction();
/**
+ * @return Affinity function for test.
+ */
+ protected abstract AffinityFunction affinityFunctionWithAffinityBackupFilter();
+
+ /**
* @throws Exception If failed.
*/
public void testPartitionDistribution() throws Exception {
+ backups = 1;
try {
for (int i = 0; i < 3; i++) {
splitAttrVal = "A";
@@ -135,4 +199,61 @@ public abstract class AffinityFunctionBackupFilterAbstractSelfTest extends GridC
assertFalse(F.eq(primary.attribute(SPLIT_ATTRIBUTE_NAME), backup.attribute(SPLIT_ATTRIBUTE_NAME)));
}
}
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testPartitionDistributionWithAffinityBackupFilter() throws Exception {
+ backups = 3;
+ try {
+ for (int i = 0; i < 2; i++) {
+ splitAttrVal = FIRST_NODE_GROUP;
+
+ startGrid(4 * i);
+
+ startGrid(4 * i + 3);
+
+ splitAttrVal = "B";
+
+ startGrid(4 * i + 1);
+
+ splitAttrVal = "C";
+
+ startGrid(4 * i + 2);
+
+ awaitPartitionMapExchange();
+
+ checkPartitionsWithAffinityBackupFilter();
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ConstantConditions")
+ private void checkPartitionsWithAffinityBackupFilter() throws Exception {
+ AffinityFunction aff = cacheConfiguration(grid(0).configuration(), null).getAffinity();
+
+ int partCnt = aff.partitions();
+
+ IgniteCache<Object, Object> cache = grid(0).cache(null);
+
+ for (int i = 0; i < partCnt; i++) {
+ Collection<ClusterNode> nodes = affinity(cache).mapKeyToPrimaryAndBackups(i);
+
+ assertEquals(backups + 1, nodes.size());
+
+ Map<String, Integer> stat = getAttributeStatistic(nodes);
+
+ assertEquals(stat.get(FIRST_NODE_GROUP), new Integer(2));
+
+ assertEquals(stat.get("B"), new Integer(1));
+
+ assertEquals(stat.get("C"), new Integer(1));
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f175d3c6/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterSelfTest.java
index eedc9e4..7fddf30 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/fair/FairAffinityFunctionBackupFilterSelfTest.java
@@ -32,4 +32,13 @@ public class FairAffinityFunctionBackupFilterSelfTest extends AffinityFunctionBa
return aff;
}
+
+ /** {@inheritDoc} */
+ @Override protected AffinityFunction affinityFunctionWithAffinityBackupFilter() {
+ FairAffinityFunction aff = new FairAffinityFunction(false);
+
+ aff.setAffinityBackupFilter(affinityBackupFilter);
+
+ return aff;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f175d3c6/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionBackupFilterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionBackupFilterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionBackupFilterSelfTest.java
index d5d8b8f..a78c383 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionBackupFilterSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunctionBackupFilterSelfTest.java
@@ -32,4 +32,13 @@ public class RendezvousAffinityFunctionBackupFilterSelfTest extends AffinityFunc
return aff;
}
+
+ /** {@inheritDoc} */
+ @Override protected AffinityFunction affinityFunctionWithAffinityBackupFilter() {
+ RendezvousAffinityFunction aff = new RendezvousAffinityFunction(false);
+
+ aff.setAffinityBackupFilter(affinityBackupFilter);
+
+ return aff;
+ }
}
[2/8] ignite git commit: IGNITE-3202: Hadoop: Implemented user name
mappers.
Posted by dm...@apache.org.
IGNITE-3202: Hadoop: Implemented user name mappers.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a815067a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a815067a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a815067a
Branch: refs/heads/master
Commit: a815067a0e8e7cfc9f49c57efe2b2eba562e9071
Parents: 56c6056
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue May 31 10:23:00 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue May 31 11:36:08 2016 +0300
----------------------------------------------------------------------
.../hadoop/fs/BasicHadoopFileSystemFactory.java | 56 +++++++-
.../fs/CachingHadoopFileSystemFactory.java | 7 +-
.../fs/KerberosHadoopFileSystemFactory.java | 4 +-
.../ignite/hadoop/util/BasicUserNameMapper.java | 114 +++++++++++++++
.../hadoop/util/ChainedUserNameMapper.java | 94 +++++++++++++
.../hadoop/util/KerberosUserNameMapper.java | 137 +++++++++++++++++++
.../ignite/hadoop/util/UserNameMapper.java | 37 +++++
.../util/BasicUserNameMapperSelfTest.java | 116 ++++++++++++++++
.../util/ChainedUserNameMapperSelfTest.java | 90 ++++++++++++
.../util/KerberosUserNameMapperSelfTest.java | 99 ++++++++++++++
.../ignite/igfs/Hadoop1DualAbstractTest.java | 46 +++++++
.../testsuites/IgniteHadoopTestSuite.java | 8 ++
12 files changed, 796 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
index 01fe6c9..06f76c3 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
@@ -21,6 +21,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.ignite.IgniteException;
import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
+import org.apache.ignite.hadoop.util.KerberosUserNameMapper;
+import org.apache.ignite.hadoop.util.UserNameMapper;
import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -46,10 +48,13 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
private static final long serialVersionUID = 0L;
/** File system URI. */
- protected String uri;
+ private String uri;
/** File system config paths. */
- protected String[] cfgPaths;
+ private String[] cfgPaths;
+
+ /** User name mapper. */
+ private UserNameMapper usrNameMapper;
/** Configuration of the secondary filesystem, never null. */
protected transient Configuration cfg;
@@ -65,8 +70,13 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
}
/** {@inheritDoc} */
- @Override public FileSystem get(String usrName) throws IOException {
- return get0(IgfsUtils.fixUserName(usrName));
+ @Override public final FileSystem get(String name) throws IOException {
+ String name0 = IgfsUtils.fixUserName(name);
+
+ if (usrNameMapper != null)
+ name0 = IgfsUtils.fixUserName(usrNameMapper.map(name0));
+
+ return getWithMappedName(name0);
}
/**
@@ -76,7 +86,7 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
* @return File system.
* @throws IOException If failed.
*/
- protected FileSystem get0(String usrName) throws IOException {
+ protected FileSystem getWithMappedName(String usrName) throws IOException {
assert cfg != null;
try {
@@ -168,6 +178,32 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
this.cfgPaths = cfgPaths;
}
+ /**
+ * Get optional user name mapper.
+ * <p>
+ * When IGFS is invoked from Hadoop, user name is passed along the way to ensure that request will be performed
+ * with proper user context. User name is passed in a simple form and doesn't contain any extended information,
+ * such as host, domain or Kerberos realm. You may use name mapper to translate plain user name to full user
+ * name required by security engine of the underlying file system.
+ * <p>
+ * For example you may want to use {@link KerberosUserNameMapper} to user name from {@code "johndoe"} to
+ * {@code "johndoe@YOUR.REALM.COM"}.
+ *
+ * @return User name mapper.
+ */
+ @Nullable public UserNameMapper getUserNameMapper() {
+ return usrNameMapper;
+ }
+
+ /**
+ * Set optional user name mapper. See {@link #getUserNameMapper()} for more information.
+ *
+ * @param usrNameMapper User name mapper.
+ */
+ public void setUserNameMapper(@Nullable UserNameMapper usrNameMapper) {
+ this.usrNameMapper = usrNameMapper;
+ }
+
/** {@inheritDoc} */
@Override public void start() throws IgniteException {
cfg = HadoopUtils.safeCreateConfiguration();
@@ -201,11 +237,15 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
throw new IgniteException("Failed to resolve secondary file system URI: " + uri);
}
}
+
+ if (usrNameMapper != null && usrNameMapper instanceof LifecycleAware)
+ ((LifecycleAware)usrNameMapper).start();
}
/** {@inheritDoc} */
@Override public void stop() throws IgniteException {
- // No-op.
+ if (usrNameMapper != null && usrNameMapper instanceof LifecycleAware)
+ ((LifecycleAware)usrNameMapper).stop();
}
/** {@inheritDoc} */
@@ -220,6 +260,8 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
}
else
out.writeInt(-1);
+
+ out.writeObject(usrNameMapper);
}
/** {@inheritDoc} */
@@ -234,5 +276,7 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Ex
for (int i = 0; i < cfgPathsCnt; i++)
cfgPaths[i] = U.readString(in);
}
+
+ usrNameMapper = (UserNameMapper)in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
index e1b30c4..bcbb082 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
@@ -23,7 +23,6 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemsUtils;
import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap;
-import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import java.io.IOException;
import java.net.URI;
@@ -47,7 +46,7 @@ public class CachingHadoopFileSystemFactory extends BasicHadoopFileSystemFactory
private final transient HadoopLazyConcurrentMap<String, FileSystem> cache = new HadoopLazyConcurrentMap<>(
new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
@Override public FileSystem createValue(String key) throws IOException {
- return get0(key);
+ return CachingHadoopFileSystemFactory.super.getWithMappedName(key);
}
}
);
@@ -60,8 +59,8 @@ public class CachingHadoopFileSystemFactory extends BasicHadoopFileSystemFactory
}
/** {@inheritDoc} */
- @Override public FileSystem get(String usrName) throws IOException {
- return cache.getOrCreate(IgfsUtils.fixUserName(usrName));
+ @Override public FileSystem getWithMappedName(String name) throws IOException {
+ return cache.getOrCreate(name);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/KerberosHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/KerberosHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/KerberosHadoopFileSystemFactory.java
index a78cabc..bbfbc59 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/KerberosHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/KerberosHadoopFileSystemFactory.java
@@ -68,10 +68,10 @@ public class KerberosHadoopFileSystemFactory extends BasicHadoopFileSystemFactor
}
/** {@inheritDoc} */
- @Override public FileSystem get(String userName) throws IOException {
+ @Override public FileSystem getWithMappedName(String name) throws IOException {
reloginIfNeeded();
- return super.get(userName);
+ return super.getWithMappedName(name);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/BasicUserNameMapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/BasicUserNameMapper.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/BasicUserNameMapper.java
new file mode 100644
index 0000000..aea7196
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/BasicUserNameMapper.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Map;
+
+/**
+ * Name mapper which maps one user name to another based on predefined dictionary. If name is not found in the
+ * dictionary, or dictionary is not defined, either passed user name or some default value could be returned.
+ */
+public class BasicUserNameMapper implements UserNameMapper {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Mappings. */
+ private Map<String, String> mappings;
+
+ /** Whether to use default user name. */
+ private boolean useDfltUsrName;;
+
+ /** Default user name. */
+ private String dfltUsrName;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String map(String name) {
+ assert mappings != null;
+
+ String res = mappings.get(name);
+
+ return res != null ? res : useDfltUsrName ? dfltUsrName : name;
+ }
+
+ /**
+ * Get mappings.
+ *
+ * @return Mappings.
+ */
+ @Nullable public Map<String, String> getMappings() {
+ return mappings;
+ }
+
+ /**
+ * Set mappings.
+ *
+ * @param mappings Mappings.
+ */
+ public void setMappings(@Nullable Map<String, String> mappings) {
+ this.mappings = mappings;
+ }
+
+ /**
+ * Get whether to use default user name when there is no mapping for current user name.
+ *
+ * @return Whether to use default user name.
+ */
+ public boolean isUseDefaultUserName() {
+ return useDfltUsrName;
+ }
+
+ /**
+ * Set whether to use default user name when there is no mapping for current user name.
+ *
+ * @param useDfltUsrName Whether to use default user name.
+ */
+ public void setUseDefaultUserName(boolean useDfltUsrName) {
+ this.useDfltUsrName = useDfltUsrName;
+ }
+
+ /**
+ * Get default user name (optional).
+ * <p>
+ * This user name will be used if provided mappings doesn't contain mapping for the given user name and
+ * {#isUseDefaultUserName} is set to {@code true}.
+ * <p>
+ * Defaults to {@code null}.
+ *
+ * @return Default user name.
+ */
+ @Nullable public String getDefaultUserName() {
+ return dfltUsrName;
+ }
+
+ /**
+ * Set default user name (optional). See {@link #getDefaultUserName()} for more information.
+ *
+ * @param dfltUsrName Default user name.
+ */
+ public void setDefaultUserName(@Nullable String dfltUsrName) {
+ this.dfltUsrName = dfltUsrName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(BasicUserNameMapper.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/ChainedUserNameMapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/ChainedUserNameMapper.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/ChainedUserNameMapper.java
new file mode 100644
index 0000000..7635e25
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/ChainedUserNameMapper.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lifecycle.LifecycleAware;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Arrays;
+
+/**
+ * Chained user name mapper. Delegate name conversion to child mappers.
+ */
+public class ChainedUserNameMapper implements UserNameMapper, LifecycleAware {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Child mappers. */
+ private UserNameMapper[] mappers;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String map(String name) {
+ for (UserNameMapper mapper : mappers)
+ name = mapper.map(name);
+
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ if (mappers == null)
+ throw new IgniteException("Mappers cannot be null.");
+
+ for (int i = 0; i < mappers.length; i++) {
+ if (mappers[i] == null)
+ throw new IgniteException("Mapper cannot be null [index=" + i + ']');
+ }
+
+ for (UserNameMapper mapper : mappers) {
+ if (mapper instanceof LifecycleAware)
+ ((LifecycleAware)mapper).start();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ assert mappers != null;
+
+ for (UserNameMapper mapper : mappers) {
+ if (mapper instanceof LifecycleAware)
+ ((LifecycleAware)mapper).stop();
+ }
+ }
+
+ /**
+ * Get child mappers.
+ *
+ * @return Child mappers.
+ */
+ public UserNameMapper[] getMappers() {
+ return mappers;
+ }
+
+ /**
+ * Set child mappers.
+ *
+ * @param mappers Child mappers.
+ */
+ public void setMappers(UserNameMapper... mappers) {
+ this.mappers = mappers;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ChainedUserNameMapper.class, this,
+ "mappers", mappers != null ? Arrays.toString(mappers) : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/KerberosUserNameMapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/KerberosUserNameMapper.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/KerberosUserNameMapper.java
new file mode 100644
index 0000000..433fb82
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/KerberosUserNameMapper.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lifecycle.LifecycleAware;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Kerberos user name mapper. Use it when you need to map simple user name to Kerberos principal.
+ * E.g. from {@code johndoe} to {@code johndoe@YOUR.REALM.COM} or {@code johndoe/admin@YOUR.REALM.COM}.
+ */
+public class KerberosUserNameMapper implements UserNameMapper, LifecycleAware {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Instance. */
+ private String instance;
+
+ /** Realm. */
+ private String realm;
+
+ /** State. */
+ private volatile State state;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String map(String name) {
+ assert state != null;
+
+ name = IgfsUtils.fixUserName(name);
+
+ switch (state) {
+ case NAME:
+ return name;
+
+ case NAME_REALM:
+ return name + '@' + realm;
+
+ case NAME_INSTANCE:
+ return name + '/' + instance;
+
+ default:
+ assert state == State.NAME_INSTANCE_REALM;
+
+ return name + '/' + instance + '@' + realm;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ if (!F.isEmpty(instance))
+ state = F.isEmpty(realm) ? State.NAME_INSTANCE : State.NAME_INSTANCE_REALM;
+ else
+ state = F.isEmpty(realm) ? State.NAME : State.NAME_REALM;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ // No-op.
+ }
+
+ /**
+ * Get Kerberos instance (optional).
+ *
+ * @return Instance.
+ */
+ @Nullable public String getInstance() {
+ return instance;
+ }
+
+ /**
+ * Set Kerberos instance (optional).
+ *
+ * @param instance Kerberos instance.
+ */
+ public void setInstance(@Nullable String instance) {
+ this.instance = instance;
+ }
+
+ /**
+ * Get Kerberos realm (optional).
+ *
+ * @return Kerberos realm.
+ */
+ @Nullable public String getRealm() {
+ return realm;
+ }
+
+ /**
+ * Set Kerberos realm (optional).
+ *
+ * @param realm Kerberos realm.
+ */
+ public void setRealm(@Nullable String realm) {
+ this.realm = realm;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(KerberosUserNameMapper.class, this);
+ }
+
+ /**
+ * State enumeration.
+ */
+ private enum State {
+ /** Name only. */
+ NAME,
+
+ /** Name and realm. */
+ NAME_REALM,
+
+ /** Name and host. */
+ NAME_INSTANCE,
+
+ /** Name, host and realm. */
+ NAME_INSTANCE_REALM,
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/UserNameMapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/UserNameMapper.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/UserNameMapper.java
new file mode 100644
index 0000000..26dc4b2
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/UserNameMapper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.Serializable;
+
+/**
+ * Hadoop file system name mapper. Used by {@link HadoopFileSystemFactory} implementation to pass proper user names
+ * to the underlying Hadoop file system.
+ */
+public interface UserNameMapper extends Serializable {
+ /**
+ * Map user name.
+ *
+ * @param name User name.
+ * @return Mapped user name.
+ */
+ @Nullable public String map(String name);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/BasicUserNameMapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/BasicUserNameMapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/BasicUserNameMapperSelfTest.java
new file mode 100644
index 0000000..54b03f9
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/BasicUserNameMapperSelfTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test for basic user name mapper.
+ */
+public class BasicUserNameMapperSelfTest extends GridCommonAbstractTest {
+ /**
+ * Test empty mappings.
+ *
+ * @throws Exception If failed.
+ */
+ public void testEmptyMappings() throws Exception {
+ Map<String, String> map = new HashMap<>();
+
+ BasicUserNameMapper mapper = create(map, false, null);
+
+ assertNull(mapper.map(null));
+ assertEquals("1", mapper.map("1"));
+ assertEquals("2", mapper.map("2"));
+
+ mapper = create(map, true, null);
+
+ assertNull(mapper.map(null));
+ assertNull(mapper.map("1"));
+ assertNull(mapper.map("2"));
+
+ mapper = create(map, false, "A");
+
+ assertNull(mapper.map(null));
+ assertEquals("1", mapper.map("1"));
+ assertEquals("2", mapper.map("2"));
+
+ mapper = create(map, true, "A");
+
+ assertEquals("A", mapper.map(null));
+ assertEquals("A", mapper.map("1"));
+ assertEquals("A", mapper.map("2"));
+ }
+
+ /**
+ * Test regular mappings.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMappings() throws Exception {
+ Map<String, String> map = new HashMap<>();
+
+ map.put("1", "101");
+
+ BasicUserNameMapper mapper = create(map, false, null);
+
+ assertNull(mapper.map(null));
+ assertEquals("101", mapper.map("1"));
+ assertEquals("2", mapper.map("2"));
+
+ mapper = create(map, true, null);
+
+ assertNull(mapper.map(null));
+ assertEquals("101", mapper.map("1"));
+ assertNull(mapper.map("2"));
+
+ mapper = create(map, false, "A");
+
+ assertNull(mapper.map(null));
+ assertEquals("101", mapper.map("1"));
+ assertEquals("2", mapper.map("2"));
+
+ mapper = create(map, true, "A");
+
+ assertEquals("A", mapper.map(null));
+ assertEquals("101", mapper.map("1"));
+ assertEquals("A", mapper.map("2"));
+ }
+
+ /**
+ * Create mapper.
+ *
+ * @param dictionary Dictionary.
+ * @param useDfltUsrName Whether to use default user name.
+ * @param dfltUsrName Default user name.
+ * @return Mapper.
+ */
+ private BasicUserNameMapper create(@Nullable Map<String, String> dictionary, boolean useDfltUsrName,
+ @Nullable String dfltUsrName) {
+ BasicUserNameMapper mapper = new BasicUserNameMapper();
+
+ mapper.setMappings(dictionary);
+ mapper.setUseDefaultUserName(useDfltUsrName);
+ mapper.setDefaultUserName(dfltUsrName);
+
+ return mapper;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
new file mode 100644
index 0000000..5bdac8f
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
@@ -0,0 +1,90 @@
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.util.Collections;
+import java.util.concurrent.Callable;
+
+/**
+ * Tests for chained user name mapper.
+ */
+public class ChainedUserNameMapperSelfTest extends GridCommonAbstractTest {
+ /** Test instance. */
+ private static final String INSTANCE = "test_instance";
+
+ /** Test realm. */
+ private static final String REALM = "test_realm";
+
+ /**
+ * Test case when mappers are null.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testNullMappers() throws Exception {
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ create((UserNameMapper[])null);
+
+ return null;
+ }
+ }, IgniteException.class, null);
+ }
+
+ /**
+ * Test case when one of mappers is null.
+ *
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ public void testNullMapperElement() throws Exception {
+ GridTestUtils.assertThrows(null, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ create(new BasicUserNameMapper(), null);
+
+ return null;
+ }
+ }, IgniteException.class, null);
+ }
+
+ /**
+ * Test actual chaining logic.
+ *
+ * @throws Exception If failed.
+ */
+ public void testChaining() throws Exception {
+ BasicUserNameMapper mapper1 = new BasicUserNameMapper();
+
+ mapper1.setMappings(Collections.singletonMap("1", "101"));
+
+ KerberosUserNameMapper mapper2 = new KerberosUserNameMapper();
+
+ mapper2.setInstance(INSTANCE);
+ mapper2.setRealm(REALM);
+
+ ChainedUserNameMapper mapper = create(mapper1, mapper2);
+
+ assertEquals("101" + "/" + INSTANCE + "@" + REALM, mapper.map("1"));
+ assertEquals("2" + "/" + INSTANCE + "@" + REALM, mapper.map("2"));
+ assertEquals(IgfsUtils.fixUserName(null) + "/" + INSTANCE + "@" + REALM, mapper.map(null));
+ }
+
+ /**
+ * Create chained mapper.
+ *
+ * @param mappers Child mappers.
+ * @return Chained mapper.
+ */
+ private ChainedUserNameMapper create(UserNameMapper... mappers) {
+ ChainedUserNameMapper mapper = new ChainedUserNameMapper();
+
+ mapper.setMappers(mappers);
+
+ mapper.start();
+
+ return mapper;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/KerberosUserNameMapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/KerberosUserNameMapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/KerberosUserNameMapperSelfTest.java
new file mode 100644
index 0000000..cc685bb
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/KerberosUserNameMapperSelfTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.util;
+
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests for Kerberos name mapper.
+ */
+public class KerberosUserNameMapperSelfTest extends GridCommonAbstractTest {
+ /** Test instance. */
+ private static final String INSTANCE = "test_instance";
+
+ /** Test realm. */
+ private static final String REALM = "test_realm";
+
+ /**
+ * Test mapper without instance and realm components.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMapper() throws Exception {
+ KerberosUserNameMapper mapper = create(null, null);
+
+ assertEquals(IgfsUtils.fixUserName(null), mapper.map(null));
+ assertEquals("test", mapper.map("test"));
+ }
+
+ /**
+ * Test mapper with instance component.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMapperInstance() throws Exception {
+ KerberosUserNameMapper mapper = create(INSTANCE, null);
+
+ assertEquals(IgfsUtils.fixUserName(null) + "/" + INSTANCE, mapper.map(null));
+ assertEquals("test" + "/" + INSTANCE, mapper.map("test"));
+ }
+
+ /**
+ * Test mapper with realm.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMapperRealm() throws Exception {
+ KerberosUserNameMapper mapper = create(null, REALM);
+
+ assertEquals(IgfsUtils.fixUserName(null) + "@" + REALM, mapper.map(null));
+ assertEquals("test" + "@" + REALM, mapper.map("test"));
+ }
+
+ /**
+ * Test mapper with instance and realm components.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMapperInstanceAndRealm() throws Exception {
+ KerberosUserNameMapper mapper = create(INSTANCE, REALM);
+
+ assertEquals(IgfsUtils.fixUserName(null) + "/" + INSTANCE + "@" + REALM, mapper.map(null));
+ assertEquals("test" + "/" + INSTANCE + "@" + REALM, mapper.map("test"));
+ }
+
+ /**
+ * Create mapper.
+ *
+ * @param instance Instance.
+ * @param realm Realm.
+ * @return Mapper.
+ */
+ private KerberosUserNameMapper create(@Nullable String instance, @Nullable String realm) {
+ KerberosUserNameMapper mapper = new KerberosUserNameMapper();
+
+ mapper.setInstance(instance);
+ mapper.setRealm(realm);
+
+ mapper.start();
+
+ return mapper;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
index d1d7c10..4ee0b75 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1DualAbstractTest.java
@@ -19,11 +19,17 @@ package org.apache.ignite.igfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory;
import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
+import org.apache.ignite.hadoop.util.ChainedUserNameMapper;
+import org.apache.ignite.hadoop.util.KerberosUserNameMapper;
+import org.apache.ignite.hadoop.util.UserNameMapper;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.internal.processors.igfs.IgfsDualAbstractSelfTest;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.lifecycle.LifecycleAware;
+import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
import static org.apache.ignite.igfs.HadoopSecondaryFileSystemConfigurationTest.SECONDARY_CFG_PATH;
@@ -75,10 +81,21 @@ public abstract class Hadoop1DualAbstractTest extends IgfsDualAbstractSelfTest {
prepareConfiguration();
+ KerberosUserNameMapper mapper1 = new KerberosUserNameMapper();
+
+ mapper1.setRealm("TEST.COM");
+
+ TestUserNameMapper mapper2 = new TestUserNameMapper();
+
+ ChainedUserNameMapper mapper = new ChainedUserNameMapper();
+
+ mapper.setMappers(mapper1, mapper2);
+
CachingHadoopFileSystemFactory factory = new CachingHadoopFileSystemFactory();
factory.setUri(secondaryUri);
factory.setConfigPaths(secondaryConfFullPath);
+ factory.setUserNameMapper(mapper);
IgniteHadoopIgfsSecondaryFileSystem second = new IgniteHadoopIgfsSecondaryFileSystem();
@@ -110,4 +127,33 @@ public abstract class Hadoop1DualAbstractTest extends IgfsDualAbstractSelfTest {
secondaryUri = mkUri(IGFS_SCHEME, SECONDARY_AUTHORITY);
}
+
+ /**
+ * Test user name mapper.
+ */
+ private static class TestUserNameMapper implements UserNameMapper, LifecycleAware {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Started flag. */
+ private boolean started;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public String map(String name) {
+ assert started;
+ assert name != null && name.contains("@");
+
+ return name.substring(0, name.indexOf("@"));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ started = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ // No-op.
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/a815067a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 554cbc7..7d877ab 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -26,6 +26,10 @@ import org.apache.ignite.client.hadoop.HadoopClientProtocolEmbeddedSelfTest;
import org.apache.ignite.client.hadoop.HadoopClientProtocolSelfTest;
import org.apache.ignite.hadoop.cache.HadoopTxConfigCacheTest;
import org.apache.ignite.hadoop.fs.KerberosHadoopFileSystemFactorySelfTest;
+import org.apache.ignite.hadoop.util.BasicUserNameMapperSelfTest;
+import org.apache.ignite.hadoop.util.ChainedUserNameMapperSelfTest;
+import org.apache.ignite.hadoop.util.KerberosUserNameMapper;
+import org.apache.ignite.hadoop.util.KerberosUserNameMapperSelfTest;
import org.apache.ignite.igfs.Hadoop1OverIgfsDualAsyncTest;
import org.apache.ignite.igfs.Hadoop1OverIgfsDualSyncTest;
import org.apache.ignite.igfs.HadoopFIleSystemFactorySelfTest;
@@ -106,6 +110,10 @@ public class IgniteHadoopTestSuite extends TestSuite {
TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
+ suite.addTest(new TestSuite(ldr.loadClass(BasicUserNameMapperSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(KerberosUserNameMapperSelfTest.class.getName())));
+ suite.addTest(new TestSuite(ldr.loadClass(ChainedUserNameMapperSelfTest.class.getName())));
+
suite.addTest(new TestSuite(ldr.loadClass(KerberosHadoopFileSystemFactorySelfTest.class.getName())));
suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyTest.class.getName())));
[5/8] ignite git commit: Added pacajge info for
java.org.apache.ignite.hadoop.util package.
Posted by dm...@apache.org.
Added pacajge info for java.org.apache.ignite.hadoop.util package.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/21bdf314
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/21bdf314
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/21bdf314
Branch: refs/heads/master
Commit: 21bdf3148f05481d86e77e13b282ac44c6c3a059
Parents: 421cf03
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue May 31 13:54:32 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue May 31 14:03:30 2016 +0300
----------------------------------------------------------------------
.../apache/ignite/hadoop/util/package-info.java | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/21bdf314/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/package-info.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/package-info.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/package-info.java
new file mode 100644
index 0000000..d84c0ba
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Ignite Hadoop Accelerator utility classes.
+ */
+package org.apache.ignite.hadoop.util;
\ No newline at end of file
[6/8] ignite git commit: Merge remote-tracking branch
'upstream/gridgain-7.6.1' into gridgain-7.6.1
Posted by dm...@apache.org.
Merge remote-tracking branch 'upstream/gridgain-7.6.1' into gridgain-7.6.1
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3ca9febe
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3ca9febe
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3ca9febe
Branch: refs/heads/master
Commit: 3ca9febeb2bf53d17b6dea739938cff40c5fa67d
Parents: 21bdf31 167e477
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue May 31 14:03:48 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue May 31 14:03:48 2016 +0300
----------------------------------------------------------------------
.../IgniteTxExceptionAbstractSelfTest.java | 43 ++++++++++++--------
1 file changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[4/8] ignite git commit: IGNITE-2708: fix tests that don't create
unique IndexingSPI per node (cherry picked from commit f90e424)
Posted by dm...@apache.org.
IGNITE-2708: fix tests that don't create unique IndexingSPI per node
(cherry picked from commit f90e424)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/167e4778
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/167e4778
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/167e4778
Branch: refs/heads/master
Commit: 167e47781f65c056b9d273eea52d268bc9a8fa0b
Parents: 421cf03
Author: Denis Magda <dm...@gridgain.com>
Authored: Tue May 31 13:50:06 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Tue May 31 13:55:35 2016 +0300
----------------------------------------------------------------------
.../IgniteTxExceptionAbstractSelfTest.java | 43 ++++++++++++--------
1 file changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/167e4778/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
index 4826c68..c867d52 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
@@ -28,6 +28,7 @@ import javax.cache.processor.EntryProcessor;
import javax.cache.processor.MutableEntry;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -54,9 +55,6 @@ import static org.apache.ignite.cache.CacheMode.REPLICATED;
* Tests that transaction is invalidated in case of {@link IgniteTxHeuristicCheckedException}.
*/
public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstractSelfTest {
- /** Index SPI throwing exception. */
- private static TestIndexingSpi idxSpi = new TestIndexingSpi();
-
/** */
private static final int PRIMARY = 0;
@@ -78,7 +76,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
- cfg.setIndexingSpi(idxSpi);
+ cfg.setIndexingSpi(new TestIndexingSpi());
cfg.getTransactionConfiguration().setTxSerializableEnabled(true);
@@ -108,7 +106,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
Transaction tx = jcache().unwrap(Ignite.class).transactions().tx();
@@ -321,7 +319,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
IgniteCache<Integer, Integer> cache = grid(0).cache(null);
if (putBefore) {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
info("Start transaction.");
@@ -344,7 +342,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
grid(i).cache(null).get(key);
}
- idxSpi.forceFail(true);
+ failIndexingSpi(true);
try {
info("Start transaction.");
@@ -380,7 +378,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
*/
@SuppressWarnings("unchecked")
private void checkUnlocked(final Integer key) throws Exception {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
awaitPartitionMapExchange();
@@ -448,7 +446,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
*/
private void checkPut(boolean putBefore, final Integer key) throws Exception {
if (putBefore) {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
info("Put key: " + key);
@@ -459,7 +457,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
for (int i = 0; i < gridCount(); i++)
grid(i).cache(null).get(key);
- idxSpi.forceFail(true);
+ failIndexingSpi(true);
info("Going to put: " + key);
@@ -481,7 +479,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
*/
private void checkTransform(boolean putBefore, final Integer key) throws Exception {
if (putBefore) {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
info("Put key: " + key);
@@ -492,7 +490,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
for (int i = 0; i < gridCount(); i++)
grid(i).cache(null).get(key);
- idxSpi.forceFail(true);
+ failIndexingSpi(true);
info("Going to transform: " + key);
@@ -524,7 +522,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
assert keys.length > 1;
if (putBefore) {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
Map<Integer, Integer> m = new HashMap<>();
@@ -542,7 +540,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
grid(i).cache(null).get(key);
}
- idxSpi.forceFail(true);
+ failIndexingSpi(true);
final Map<Integer, Integer> m = new HashMap<>();
@@ -570,7 +568,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
*/
private void checkRemove(boolean putBefore, final Integer key) throws Exception {
if (putBefore) {
- idxSpi.forceFail(false);
+ failIndexingSpi(false);
info("Put key: " + key);
@@ -581,7 +579,7 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
for (int i = 0; i < gridCount(); i++)
grid(i).cache(null).get(key);
- idxSpi.forceFail(true);
+ failIndexingSpi(true);
info("Going to remove: " + key);
@@ -653,6 +651,19 @@ public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstrac
}
/**
+ * Controls indexing SPI behavior.
+ *
+ * @param fail fail the SPI or not.
+ */
+ private void failIndexingSpi(boolean fail) {
+ for (Ignite ignite : Ignition.allGrids()) {
+ TestIndexingSpi idxSpi = (TestIndexingSpi)ignite.configuration().getIndexingSpi();
+
+ idxSpi.forceFail(fail);
+ }
+ }
+
+ /**
* Indexing SPI that can fail on demand.
*/
private static class TestIndexingSpi extends IgniteSpiAdapter implements IndexingSpi {
[3/8] ignite git commit: 7.5.24: Added missing JavaDocs.
Posted by dm...@apache.org.
7.5.24: Added missing JavaDocs.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/421cf03b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/421cf03b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/421cf03b
Branch: refs/heads/master
Commit: 421cf03b03276d0ab6c32ef6665d547746446dfc
Parents: a815067
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue May 31 10:40:24 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue May 31 11:36:47 2016 +0300
----------------------------------------------------------------------
.../hadoop/util/ChainedUserNameMapperSelfTest.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/421cf03b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
index 5bdac8f..bfac49c 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/hadoop/util/ChainedUserNameMapperSelfTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.hadoop.util;
import org.apache.ignite.IgniteException;
[8/8] ignite git commit: Merge remote-tracking branch
'remotes/community/gridgain-7.6.1'
Posted by dm...@apache.org.
Merge remote-tracking branch 'remotes/community/gridgain-7.6.1'
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4cc10982
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4cc10982
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4cc10982
Branch: refs/heads/master
Commit: 4cc10982e06fa521ed93f5930b2ccddcce405c56
Parents: 3c3ed05 f175d3c
Author: Denis Magda <dm...@gridgain.com>
Authored: Thu Jun 2 16:25:43 2016 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu Jun 2 16:25:43 2016 +0300
----------------------------------------------------------------------
.../affinity/fair/FairAffinityFunction.java | 81 ++++++++++-
.../rendezvous/RendezvousAffinityFunction.java | 39 +++++-
.../apache/ignite/internal/util/ClassCache.java | 32 +++++
.../ignite/internal/util/IgniteUtils.java | 6 +-
...ityFunctionBackupFilterAbstractSelfTest.java | 131 +++++++++++++++++-
...airAffinityFunctionBackupFilterSelfTest.java | 9 ++
...ousAffinityFunctionBackupFilterSelfTest.java | 9 ++
.../IgniteTxExceptionAbstractSelfTest.java | 3 +-
.../hadoop/fs/BasicHadoopFileSystemFactory.java | 56 +++++++-
.../fs/CachingHadoopFileSystemFactory.java | 7 +-
.../fs/KerberosHadoopFileSystemFactory.java | 4 +-
.../ignite/hadoop/util/BasicUserNameMapper.java | 114 +++++++++++++++
.../hadoop/util/ChainedUserNameMapper.java | 94 +++++++++++++
.../hadoop/util/KerberosUserNameMapper.java | 137 +++++++++++++++++++
.../ignite/hadoop/util/UserNameMapper.java | 37 +++++
.../apache/ignite/hadoop/util/package-info.java | 22 +++
.../processors/hadoop/HadoopClassLoader.java | 23 +++-
.../util/BasicUserNameMapperSelfTest.java | 116 ++++++++++++++++
.../util/ChainedUserNameMapperSelfTest.java | 107 +++++++++++++++
.../util/KerberosUserNameMapperSelfTest.java | 99 ++++++++++++++
.../ignite/igfs/Hadoop1DualAbstractTest.java | 46 +++++++
.../testsuites/IgniteHadoopTestSuite.java | 8 ++
22 files changed, 1156 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4cc10982/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
----------------------------------------------------------------------
diff --cc modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
index 95f1740,c867d52..6c7f7ac
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxExceptionAbstractSelfTest.java
@@@ -53,7 -54,7 +54,7 @@@ import static org.apache.ignite.cache.C
/**
* Tests that transaction is invalidated in case of {@link IgniteTxHeuristicCheckedException}.
*/
--public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstractSelfTest {
++public abstract class IgniteTxExceptionAbstractSelfTest extends GridCacheAbstractSelfTest
/** */
private static final int PRIMARY = 0;