You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by kl...@apache.org on 2016/03/18 22:43:26 UTC
[01/14] incubator-geode git commit: GEODE-1044: InitialImagePut
executes BEFORE_UPDATE_OP on indexes for updates
Repository: incubator-geode
Updated Branches:
refs/heads/feature/GEODE-1050 9c8f1cde1 -> ae7bf1642
GEODE-1044: InitialImagePut executes BEFORE_UPDATE_OP on indexes for updates
When entries are recovered from persistence, the values are added to the index,
however a gii can then occur where the initialImagePut needs to properly
alert the index that an update is occuring. The BEFORE_UPDATE_OP allows the index
to retreive the oldKey and notifies the index that an update is occuring.
Refactor QueryTestUtils and removed unused code
Minor modification for QueryTestUtils to be more generic and not be specific to Portfolios
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/54931a52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/54931a52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/54931a52
Branch: refs/heads/feature/GEODE-1050
Commit: 54931a52ed3f2c3d72156f830edc637f00b7a8b1
Parents: 8c74d60
Author: Jason Huynh <hu...@gmail.com>
Authored: Wed Mar 9 13:32:55 2016 -0800
Committer: Jason Huynh <hu...@gmail.com>
Committed: Fri Mar 18 13:24:38 2016 -0700
----------------------------------------------------------------------
.../internal/cache/AbstractRegionMap.java | 17 +-
.../gemfire/cache/query/QueryTestUtils.java | 335 ++++---------------
.../cache/query/QueryTestUtilsJUnitTest.java | 31 +-
.../query/dunit/CompactRangeIndexDUnitTest.java | 33 +-
.../cache/query/dunit/HashIndexDUnitTest.java | 29 +-
...itionedRegionCompactRangeIndexDUnitTest.java | 130 +++++++
.../index/CompactRangeIndexJUnitTest.java | 17 +-
.../dunit/PersistentPartitionWithIndex.xml | 17 +
8 files changed, 314 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java
index 9058984..9497131 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java
@@ -954,6 +954,11 @@ public abstract class AbstractRegionMap implements RegionMap {
}
}
if (owner.getIndexManager() != null) {
+ // Due to having no reverse map, we need to be able to generate the oldkey before doing an update
+ // Without the BEFORE_UPDATE_OP, we would see duplicate entries in the index as the update could not locate the old key
+ if (!oldRe.isRemoved()) {
+ owner.getIndexManager().updateIndexes(oldRe, IndexManager.REMOVE_ENTRY, IndexProtocol.BEFORE_UPDATE_OP);
+ }
owner.getIndexManager().updateIndexes(oldRe, oldRe.isRemoved() ? IndexManager.ADD_ENTRY : IndexManager.UPDATE_ENTRY,
oldRe.isRemoved() ? IndexProtocol.OTHER_OP : IndexProtocol.AFTER_UPDATE_OP);
}
@@ -992,10 +997,16 @@ public abstract class AbstractRegionMap implements RegionMap {
}
incEntryCount(1);
}
+
//Update local indexes
- if (owner.getIndexManager() != null) {
- owner.getIndexManager().updateIndexes(newRe, newRe.isRemoved() ? IndexManager.REMOVE_ENTRY : IndexManager.UPDATE_ENTRY,
- newRe.isRemoved() ? IndexProtocol.OTHER_OP : IndexProtocol.AFTER_UPDATE_OP);
+ if (owner.getIndexManager() != null) {
+ // Due to having no reverse map, we need to be able to generate the oldkey before doing an update
+ // Without the BEFORE_UPDATE_OP, we would see duplicate entries in the index as the update could not locate the old key
+ if (oldRe != null && !oldRe.isRemoved()) {
+ owner.getIndexManager().updateIndexes(oldRe, IndexManager.REMOVE_ENTRY, IndexProtocol.BEFORE_UPDATE_OP);
+ }
+ owner.getIndexManager().updateIndexes(newRe, newRe.isRemoved() ? IndexManager.REMOVE_ENTRY : IndexManager.UPDATE_ENTRY,
+ newRe.isRemoved() ? IndexProtocol.OTHER_OP : IndexProtocol.AFTER_UPDATE_OP);
}
done = true;
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtils.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtils.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtils.java
index ce7c9fd..edfe6b3 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtils.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtils.java
@@ -16,6 +16,8 @@
*/
package com.gemstone.gemfire.cache.query;
+import java.io.File;
+import java.io.IOException;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
@@ -56,14 +58,16 @@ public class QueryTestUtils implements Serializable {
private static Cache cache;
- private static QueryService qs;
-
- public static final String KEY = "key-";
-
public HashMap<String, String> queries;
public HashMap<String, String> bindQueries;
+ private static QueryTestUtils instance = new QueryTestUtils();
+
+ public static QueryTestUtils getInstance() {
+ return instance;
+ }
+
public QueryTestUtils() {
queries = new HashMap<String, String>();
bindQueries = new HashMap<String, String>();
@@ -1036,6 +1040,10 @@ public class QueryTestUtils implements Serializable {
bindQueries.put("81", "select p from /exampleRegion.values p where p like $1");
bindQueries.put("82", "select p.positions.get('acc') from $1 p");
}
+
+ public static QueryService getQueryService() {
+ return cache.getQueryService();
+ }
public void createServer(VM server, final Properties prop) {
SerializableRunnable createCacheServer = new CacheSerializableRunnable(
@@ -1061,27 +1069,6 @@ public class QueryTestUtils implements Serializable {
public static void setCache(Cache cache) {
QueryTestUtils.cache = cache;
}
-
- public void createClient(VM client, final Properties prop) {
- SerializableRunnable createCacheClient = new CacheSerializableRunnable(
- "Create Cache Client") {
- private static final long serialVersionUID = 1L;
-
- public void run2() throws CacheException {
- createClientCache(prop);
- }
- };
- client.invoke(createCacheClient);
- }
-
- public void createClientCache(Properties prop) {
- if (null != prop && !prop.isEmpty()) {
- cache = (Cache) new ClientCacheFactory(prop).create();
- }
- else{
- cache = (Cache) new ClientCacheFactory().create();
- }
- }
public void createPartitionRegion(final String name, final Class constraint, VM vm) {
vm.invoke(new CacheSerializableRunnable("Create Partition region") {
@@ -1159,7 +1146,7 @@ public class QueryTestUtils implements Serializable {
public void removeIndex(String name, String region){
try {
- qs = cache.getQueryService();
+ QueryService qs = getQueryService();
qs.removeIndex(qs.getIndex(cache.getRegion(region), name));
} catch (Exception e) {
e.printStackTrace();
@@ -1219,99 +1206,6 @@ public class QueryTestUtils implements Serializable {
cache.close();
}
}
-
- /**
- * Places new values in the given region with Object ({@link Portfolio}) Keys
- * @param vm
- * @param regionName
- * @param size
- */
- public void createValuesObjKeys(VM vm, final String regionName, final int size) {
- vm.invoke(new SerializableRunnable() {
- public void run() {
- createValuesObjKeys(regionName, size);
- }
- });
- }
-
- /**
- * Places new values in the given region with {@link String} Keys
- * @param vm
- * @param regionName
- * @param size
- */
- public void createValuesStringKeys(VM vm, final String regionName, final int size) {
- vm.invoke(new SerializableRunnable() {
- public void run() {
- createValuesStringKeys(regionName, size);
- }
- });
- }
-
- /**
- * Places new values in the given region with {@link String} Keys
- * @param regionName
- * @param size
- */
- public void createValuesStringKeys(String regionName, final int size) {
- Region region = cache.getRegion(regionName);
- for (int i = 1; i <= size; i++) {
- region.put("KEY-"+ i, new Portfolio(i));
- }
- }
-
- /**
- * Used to update the values in the given region having {@link String} Keys
- * with modifed values. Should preferably use after the call to createValuesStringKeys
- * @param regionName
- * @param size
- */
- public void createDiffValuesStringKeys(String regionName, final int size) {
- Region region = cache.getRegion(regionName);
- for (int i = 1; i <= size; i++) {
- region.put("KEY-"+ i, new Portfolio(i+1));
- }
- }
- /**
- * Places new values in the given region with Object ({@link Portfolio}) Keys
- * @param regionName
- * @param size
- */
- public void createValuesObjKeys(String regionName, final int size) {
- Region region = cache.getRegion(regionName);
- for (int i = 1; i <= size; i++) {
- region.put(new Portfolio(i), new Portfolio(i));
- }
- }
-
- /**
- * Places new values in the given region with {@link String} Keys
- * @param regionName
- * @param size
- */
- public void createNumericValuesStringKeys(String regionName, final int size) {
- Region region = cache.getRegion(regionName);
- for (int i = 1; i <= size; i++) {
- region.put("KEY-"+ i, new Numbers(i));
- }
- }
-
- /**
- * Destroys entries from the region
- * @param regionName
- * @param size
- * @throws Exception
- */
- public void destroyRegion(String regionName, int size) throws Exception{
- Region region = cache.getRegion(regionName);
- for (int i = 1; i <= size; i++) {
- try {
- region.destroy("KEY-"+ i);
- } catch (Exception e) {
- throw new Exception(e);
- }
- }
- }
/**
* Executes queries corresponding to the keys passed using array<br>
@@ -1348,7 +1242,7 @@ public class QueryTestUtils implements Serializable {
* Object array containing SelectResults
*/
public Object[] executeQueries(String qarr[]) throws Exception{
- qs = cache.getQueryService();
+ QueryService qs = cache.getQueryService();
Object[] result = new Object[qarr.length];
String query = null;
int j = 0;
@@ -1377,7 +1271,7 @@ public class QueryTestUtils implements Serializable {
* @param qarr
*/
public Object[] executeQueriesWithoutDistinct(String qarr[]) {
- qs = cache.getQueryService();
+ QueryService qs = cache.getQueryService();
Object[] result = new Object[qarr.length];
String query = null;
int j = 0;
@@ -1416,7 +1310,7 @@ public class QueryTestUtils implements Serializable {
* @param qarr
*/
public Object[] executeQueriesWithDistinct(String qarr[]) {
- qs = cache.getQueryService();
+ QueryService qs = cache.getQueryService();
Object[] result = new Object[qarr.length];
String query = null;
int j = 0;
@@ -1438,40 +1332,6 @@ public class QueryTestUtils implements Serializable {
}
/**
- * Execute all the queries in the map
- * @param vm
- */
- public void executeAllQueries(VM vm) {
- vm.invoke(new CacheSerializableRunnable("Execute all queries") {
- public void run2() {
- executeAllQueries();
- }
- });
- }
-
- /**
- * Execute all the queries in the map
- */
- public Object[] executeAllQueries() {
- qs = cache.getQueryService();
- Object[] result = new Object[queries.size()];
-
- int j = 0;
- for (Map.Entry<String, String> entry : queries.entrySet()) {
- String queryId = entry.getKey();
- String query = entry.getValue();
- System.out.println("\nExecuting query: " + query);
- try {
- result[j++] = qs.newQuery(query).execute();
- formatQueryResult(result);
- } catch (Exception e) {
- throw new RuntimeException("Query: " + queryId + "-" + query + " failed: ", e);
- }
- }
- return result;
- }
-
- /**
* Executes a particular bind query corresponding to the ID passed using array<br>
* <P><code>
* String[] arr = {"1", "2", "3"};<br>
@@ -1499,7 +1359,7 @@ public class QueryTestUtils implements Serializable {
public Object executeBindQuery(final String queryId, final Object[] params) {
Object result = null;
- qs = cache.getQueryService();
+ QueryService qs = cache.getQueryService();
String query = bindQueries.get(queryId);
try {
getLogger().fine("\nExecuting query: " + query);
@@ -1510,120 +1370,8 @@ public class QueryTestUtils implements Serializable {
return result;
}
- public void compareResults(SelectResults[][] r) {
- Set set1 = null;
- Set set2 = null;
- Iterator itert1 = null;
- Iterator itert2 = null;
- ObjectType type1, type2;
-
- for (int j = 0; j < r.length; j++) {
- CollectionType collType1 = r[j][0].getCollectionType();
- CollectionType collType2 = r[j][1].getCollectionType();
- type1 = collType1.getElementType();
- type2 = collType2.getElementType();
-
- if (collType1.getSimpleClassName().equals(collType2.getSimpleClassName())) {
- CacheUtils.log("Both SelectResults are of the same Type i.e.--> "
- + collType1);
- }
- else {
- CacheUtils.log("Collection type are : " + collType1 + "and "
- + collType2);
- Assert
- .fail("FAILED:Select results Collection Type is different in both the cases. CollectionType1="
- + collType1 + " CollectionType2=" + collType2);
- }
- if (type1.equals(type2)) {
- CacheUtils.log("Both SelectResults have same element Type i.e.--> "
- + type1);
- }
- else {
- CacheUtils.log("Classes are : type1=" + type1.getSimpleClassName()
- + " type2= " + type2.getSimpleClassName());
- Assert
- .fail("FAILED:SelectResult Element Type is different in both the cases. Type1="
- + type1 + " Type2=" + type2);
- }
-
- if (collType1.equals(collType2)) {
- CacheUtils.log("Both SelectResults are of the same Type i.e.--> "
- + collType1);
- }
- else {
- CacheUtils.log("Collections are : " + collType1 + " " + collType2);
- Assert
- .fail("FAILED:SelectResults Collection Type is different in both the cases. CollType1="
- + collType1 + " CollType2=" + collType2);
- }
- if (r[j][0].size() == r[j][1].size()) {
- CacheUtils.log("Both SelectResults are of Same Size i.e. Size= "
- + r[j][1].size());
- }
- else {
- Assert
- .fail("FAILED:SelectResults size is different in both the cases. Size1="
- + r[j][0].size() + " Size2 = " + r[j][1].size());
- }
- set2 = ((r[j][1]).asSet());
- set1 = ((r[j][0]).asSet());
- // boolean pass = true;
- itert1 = set1.iterator();
- while (itert1.hasNext()) {
- Object p1 = itert1.next();
- itert2 = set2.iterator();
-
- boolean exactMatch = false;
- while (itert2.hasNext()) {
- Object p2 = itert2.next();
- if (p1 instanceof Struct) {
- Object[] values1 = ((Struct)p1).getFieldValues();
- Object[] values2 = ((Struct)p2).getFieldValues();
- Assert.assertEquals(values1.length, values2.length);
- boolean elementEqual = true;
- for (int i = 0; i < values1.length; ++i) {
- elementEqual = elementEqual
- && ((values1[i] == values2[i]) || values1[i]
- .equals(values2[i]));
- }
- exactMatch = elementEqual;
- }
- else {
- exactMatch = (p2 == p1) || p2.equals(p1);
- }
- if (exactMatch) {
- break;
- }
- }
- if (!exactMatch) {
- Assert
- .fail("Atleast one element in the pair of SelectResults supposedly identical, is not equal ");
- }
- }
- }
- }
-
- public void formatQueryResult(Object[] results) {
- log("\nResults: ");
- for (Object result : results) {
- if (result == null) {
- log("NULL");
- } else if (result == QueryService.UNDEFINED) {
- log("UNDEFINED");
- } else if (result instanceof SelectResults) {
- Collection<?> collection = ((SelectResults<?>) result).asList();
- StringBuffer sb = new StringBuffer();
- for (Object e : collection) {
- sb.append(e + "\n\t");
- }
- log(sb.toString());
- } else {
- log("Unknown");
- }
- }
- }
-
- public LogWriter getLogger() {
+
+ private LogWriter getLogger() {
if (cache == null) {
return null;
}
@@ -1633,8 +1381,47 @@ public class QueryTestUtils implements Serializable {
public Cache getCache() {
return cache;
}
-
- public void log(Object message) {
- CacheUtils.log(message);
+
+ public static void createCacheInVM(VM vm, Properties props) {
+ vm.invoke(() -> {
+ getInstance().createCache(props);
+ });
+ }
+
+ public static void closeCacheInVM(VM vm) {
+ vm.invoke(() -> {
+ getInstance().cache.close();
+ });
+ }
+
+ public Region getRegion(String regionName) {
+ return cache.getRegion(regionName);
+ }
+
+ public void populateRegion(String regionName, Map<?, ?> entries) {
+ Region r = cache.getRegion("/" + regionName);
+ entries.entrySet().forEach(e -> {
+ r.put(e.getKey(), e.getValue());
+ });
+ }
+
+ public static void populateRegion(VM vm, String regionName, Map<?, ?> entries) {
+ vm.invoke(() -> {
+ getInstance().populateRegion(regionName, entries);
+ });
+ }
+
+ public static File createTestRootDiskStore(String testName) throws IOException {
+ File diskDir = new File(testName).getAbsoluteFile();
+ com.gemstone.gemfire.internal.FileUtil.delete(diskDir);
+ diskDir.mkdir();
+ diskDir.deleteOnExit();
+ return diskDir;
+ }
+
+ public static File createRootDiskStoreInVM(VM vm, String rootDiskStoreName) {
+ return vm.invoke(() -> {
+ return createTestRootDiskStore(rootDiskStoreName);
+ });
}
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtilsJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtilsJUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtilsJUnitTest.java
index 0358aad..c5d40bd 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtilsJUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/QueryTestUtilsJUnitTest.java
@@ -30,6 +30,8 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import com.gemstone.gemfire.cache.Region;
+import com.gemstone.gemfire.cache.query.data.Numbers;
import com.gemstone.gemfire.cache.query.data.Portfolio;
import com.gemstone.gemfire.test.junit.categories.IntegrationTest;
@@ -49,14 +51,35 @@ public class QueryTestUtilsJUnitTest {
@Before
public void setUp() throws Exception {
- utils = new QueryTestUtils();
+ utils = QueryTestUtils.getInstance();
utils.createCache(null);
// create regions
utils.createReplicateRegion("exampleRegion");
utils.createReplicateRegion("numericRegion");
// put entries in the region
- utils.createValuesStringKeys("exampleRegion", 10);
- utils.createNumericValuesStringKeys("numericRegion", 10);
+ populatePortfolioValuesInRegion("exampleRegion", 10);
+ populateNumericValuesInRegion("numericRegion", 10);
+ }
+
+ private void populatePortfolioValuesInRegion(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Portfolio(i));
+ }
+ }
+
+ private void populateOffsetPortfolioValuesInRegion(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Portfolio(i + 1));
+ }
+ }
+
+ private void populateNumericValuesInRegion(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Numbers(i));
+ }
}
@Test
@@ -85,7 +108,7 @@ public class QueryTestUtilsJUnitTest {
@Test
public void testQueriesWithoutDistinct() throws Exception{
- utils.createDiffValuesStringKeys("exampleRegion", 2);
+ this.populateOffsetPortfolioValuesInRegion("exampleRegion", 2);
String[] queries = { "181" };
int results = 0;
for (Object result : utils.executeQueriesWithoutDistinct(queries)) {
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/CompactRangeIndexDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/CompactRangeIndexDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/CompactRangeIndexDUnitTest.java
index 81b545f..df655a2 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/CompactRangeIndexDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/CompactRangeIndexDUnitTest.java
@@ -19,6 +19,7 @@ package com.gemstone.gemfire.cache.query.dunit;
import java.util.Properties;
import com.gemstone.gemfire.cache.CacheException;
+import com.gemstone.gemfire.cache.Region;
import com.gemstone.gemfire.cache.query.QueryTestUtils;
import com.gemstone.gemfire.cache.query.data.Portfolio;
import com.gemstone.gemfire.cache.query.internal.index.IndexManager;
@@ -70,7 +71,7 @@ public class CompactRangeIndexDUnitTest extends DistributedTestCase{
vm0.invoke(new CacheSerializableRunnable("Putting values") {
public void run2() {
- utils.createValuesStringKeys("examplePartitionedRegion", 100);
+ putPortfolios("examplePartitionedRegion", 100);
}
});
try {
@@ -112,7 +113,7 @@ public class CompactRangeIndexDUnitTest extends DistributedTestCase{
public void doPut(final int entries) {
vm0.invokeAsync(new CacheSerializableRunnable("Putting values") {
public void run2() {
- utils.createValuesStringKeys("exampleRegion", entries);
+ putPortfolios("exampleRegion", entries);
}
});
}
@@ -120,7 +121,7 @@ public class CompactRangeIndexDUnitTest extends DistributedTestCase{
public void doPutSync(final int entries) {
vm0.invoke(new CacheSerializableRunnable("Putting values") {
public void run2() {
- utils.createValuesStringKeys("exampleRegion", entries);
+ putPortfolios("exampleRegion", entries);
}
});
}
@@ -128,7 +129,7 @@ public class CompactRangeIndexDUnitTest extends DistributedTestCase{
public void doUpdate(final int entries) {
vm0.invokeAsync(new CacheSerializableRunnable("Updating values") {
public void run2() {
- utils.createDiffValuesStringKeys("exampleRegion", entries);
+ putOffsetPortfolios("exampleRegion", entries);
}
});
}
@@ -159,7 +160,15 @@ public class CompactRangeIndexDUnitTest extends DistributedTestCase{
public void run2() {
try {
Thread.sleep(500);
- utils.destroyRegion("exampleRegion", entries);
+ //destroy entries
+ Region region = utils.getRegion("exampleRegion");
+ for (int i = 1; i <= entries; i++) {
+ try {
+ region.destroy("KEY-"+ i);
+ } catch (Exception e) {
+ throw new Exception(e);
+ }
+ }
} catch (Exception e) {
fail("Destroy failed.");
}
@@ -191,6 +200,20 @@ public class CompactRangeIndexDUnitTest extends DistributedTestCase{
});
}
+ private void putPortfolios(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Portfolio(i));
+ }
+ }
+
+ private void putOffsetPortfolios(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Portfolio(i + 1));
+ }
+ }
+
private static class CompactRangeIndexTestHook implements TestHook{
@Override
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/HashIndexDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/HashIndexDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/HashIndexDUnitTest.java
index 38721be..76f850d 100755
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/HashIndexDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/HashIndexDUnitTest.java
@@ -19,7 +19,9 @@ package com.gemstone.gemfire.cache.query.dunit;
import java.util.Properties;
import com.gemstone.gemfire.cache.CacheException;
+import com.gemstone.gemfire.cache.Region;
import com.gemstone.gemfire.cache.query.QueryTestUtils;
+import com.gemstone.gemfire.cache.query.data.Portfolio;
import com.gemstone.gemfire.cache30.CacheSerializableRunnable;
import com.gemstone.gemfire.test.dunit.DistributedTestCase;
import com.gemstone.gemfire.test.dunit.DistributedTestUtils;
@@ -76,7 +78,7 @@ public class HashIndexDUnitTest extends DistributedTestCase{
public void doPut(final int entries) {
vm0.invokeAsync(new CacheSerializableRunnable("Putting values") {
public void run2() {
- utils.createValuesStringKeys("exampleRegion", entries);
+ putPortfolios("exampleRegion", entries);
}
});
}
@@ -84,7 +86,7 @@ public class HashIndexDUnitTest extends DistributedTestCase{
public void doUpdate(final int entries) {
vm0.invokeAsync(new CacheSerializableRunnable("Updating values") {
public void run2() {
- utils.createDiffValuesStringKeys("exampleRegion", entries);
+ putOffsetPortfolios("exampleRegion", entries);
}
});
}
@@ -115,7 +117,14 @@ public class HashIndexDUnitTest extends DistributedTestCase{
e.printStackTrace();
}
try {
- utils.destroyRegion("exampleRegion", entries);
+ Region region = utils.getRegion("exampleRegion");
+ for (int i = 1; i <= entries; i++) {
+ try {
+ region.destroy("KEY-"+ i);
+ } catch (Exception e) {
+ throw new Exception(e);
+ }
+ }
}
catch (Exception e) {
throw new CacheException(e){};
@@ -129,4 +138,18 @@ public class HashIndexDUnitTest extends DistributedTestCase{
Thread.sleep(5000);
utils.closeServer(vm0);
}
+
+ private void putPortfolios(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Portfolio(i));
+ }
+ }
+
+ private void putOffsetPortfolios(String regionName, int size) {
+ Region region = utils.getRegion(regionName);
+ for (int i = 1; i <= size; i++) {
+ region.put("KEY-"+ i, new Portfolio(i + 1));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
new file mode 100644
index 0000000..8be017c
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.cache.query.dunit;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.IntStream;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.gemstone.gemfire.cache.query.Query;
+import com.gemstone.gemfire.cache.query.QueryService;
+import com.gemstone.gemfire.cache.query.QueryTestUtils;
+import com.gemstone.gemfire.cache.query.SelectResults;
+import com.gemstone.gemfire.cache.query.data.Portfolio;
+import com.gemstone.gemfire.distributed.internal.DistributionConfig;
+import com.gemstone.gemfire.test.dunit.DistributedTestCase;
+import com.gemstone.gemfire.test.dunit.DistributedTestUtils;
+import com.gemstone.gemfire.test.dunit.Host;
+import com.gemstone.gemfire.test.dunit.IgnoredException;
+import com.gemstone.gemfire.test.dunit.Invoke;
+import com.gemstone.gemfire.test.dunit.SerializableRunnable;
+import com.gemstone.gemfire.test.dunit.VM;
+import com.gemstone.gemfire.test.junit.categories.DistributedTest;
+import com.gemstone.gemfire.util.test.TestUtil;
+
+@Category(DistributedTest.class)
+public class PartitionedRegionCompactRangeIndexDUnitTest extends DistributedTestCase {
+
+ public PartitionedRegionCompactRangeIndexDUnitTest(String name) {
+ super(name);
+ }
+
+ private Properties getSystemProperties(String cacheXML) {
+ Properties props = new Properties();
+ props.setProperty(DistributionConfig.LOCATORS_NAME, "localhost["+DistributedTestUtils.getDUnitLocatorPort()+"]");
+ props.setProperty("cache-xml-file", TestUtil.getResourcePath(getClass(), cacheXML));
+ return props;
+ }
+
+ public void setUp() throws Exception {
+ this.disconnectAllFromDS();
+ }
+
+ @Override
+ protected void preTearDown() throws Exception {
+ Host host = Host.getHost(0);
+ VM vm0 = host.getVM(0);
+ VM vm1 = host.getVM(1);
+ VM vm2 = host.getVM(2);
+ QueryTestUtils.closeCacheInVM(vm0);
+ QueryTestUtils.closeCacheInVM(vm1);
+ QueryTestUtils.closeCacheInVM(vm2);
+ }
+
+ @Test
+ public void testGIIUpdateWithIndexDoesNotDuplicateEntryInIndexWhenAlreadyRecoveredFromPersistence() throws Exception {
+ Host host = Host.getHost(0);
+ VM vm0 = host.getVM(0);
+ VM vm1 = host.getVM(1);
+ VM vm2 = host.getVM(2);
+ //Adding due to known race condition for creation of partitioned indexes via cache.xml
+ IgnoredException.addIgnoredException("IndexNameConflictException");
+
+ String regionName = "persistentTestRegion"; //this region is created via cache.xml
+ String idQuery = "select * from /" + regionName + " p where p.ID = 1";
+ int idQueryExpectedSize = 1;
+ int numEntries = 100;
+ Map<String, Portfolio> entries = new HashMap<>();
+ IntStream.range(0, numEntries).forEach(i -> entries.put("key-" + i, new Portfolio(i)));
+ Map<String, Portfolio> newEntries = new HashMap<>();
+ IntStream.range(0, numEntries).forEach(i -> newEntries.put("key-" + i, new Portfolio(i + 1)));
+
+ File rootDiskStore1 = QueryTestUtils.createRootDiskStoreInVM(vm0, "diskDir-PersistentPartitionWithIndexDiskStore");
+ File rootDiskStore2 = QueryTestUtils.createRootDiskStoreInVM(vm1, "diskDir-PersistentPartitionWithIndexDiskStore");
+ File rootDiskStore3 = QueryTestUtils.createRootDiskStoreInVM(vm2, "diskDir-PersistentPartitionWithIndexDiskStore");
+
+ QueryTestUtils.createCacheInVM(vm0, getSystemProperties("PersistentPartitionWithIndex.xml"));
+ QueryTestUtils.createCacheInVM(vm1, getSystemProperties("PersistentPartitionWithIndex.xml"));
+ QueryTestUtils.createCacheInVM(vm2, getSystemProperties("PersistentPartitionWithIndex.xml"));
+ QueryTestUtils.populateRegion(vm2, regionName, entries);
+
+ vm1.invoke(verifyQueryResultsSize(idQuery, idQueryExpectedSize));
+ QueryTestUtils.closeCacheInVM(vm1);
+
+ //update entries
+ QueryTestUtils.populateRegion(vm0, regionName, entries);
+ QueryTestUtils.closeCacheInVM(vm0);
+
+ //restart 2 nodes
+ QueryTestUtils.createCacheInVM(vm0, getSystemProperties("PersistentPartitionWithIndex.xml"));
+ QueryTestUtils.createCacheInVM(vm1, getSystemProperties("PersistentPartitionWithIndex.xml"));
+
+ vm2.invoke(verifyQueryResultsSize(idQuery, idQueryExpectedSize));
+ vm1.invoke(verifyQueryResultsSize(idQuery, idQueryExpectedSize));
+ vm0.invoke(verifyQueryResultsSize(idQuery, idQueryExpectedSize));
+ }
+
+ private SerializableRunnable verifyQueryResultsSize(String query, int expectedSize) {
+ return new SerializableRunnable() {
+ public void run() {
+ try {
+ QueryService qs = QueryTestUtils.getInstance().getQueryService();
+ Query q = qs.newQuery(query);
+ SelectResults sr = (SelectResults) q.execute();
+ assertEquals(expectedSize, sr.size());
+ }
+ catch (Exception e) {
+ fail("Exception occurred when executing verifyQueryResultsSize for query:" + query);
+ }
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CompactRangeIndexJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CompactRangeIndexJUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CompactRangeIndexJUnitTest.java
index 68a946c..ba84fc8 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CompactRangeIndexJUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/internal/index/CompactRangeIndexJUnitTest.java
@@ -77,7 +77,7 @@ public class CompactRangeIndexJUnitTest {
isUsingConcurrentHashSet("type1");
utils.removeIndex("type", "/exampleRegion");
executeQueryWithAndWithoutIndex(4);
- updateValues(2);
+ putOffsetValues(2);
executeQueryWithCount();
executeQueryWithAndWithoutIndex(3);
executeRangeQueryWithDistinct(8);
@@ -377,13 +377,18 @@ public class CompactRangeIndexJUnitTest {
- public void putValues(int num) {
- long start = System.currentTimeMillis();
- utils.createValuesStringKeys("exampleRegion", num);
+ private void putValues(int num) {
+ Region region = utils.getRegion("exampleRegion");
+ for (int i = 1; i <= num; i++) {
+ region.put("KEY-"+ i, new Portfolio(i));
+ }
}
- private void updateValues(int num){
- utils.createDiffValuesStringKeys("exampleRegion", num);
+ private void putOffsetValues(int num) {
+ Region region = utils.getRegion("exampleRegion");
+ for (int i = 1; i <= num; i++) {
+ region.put("KEY-"+ i, new Portfolio(i + 1));
+ }
}
public void executeQueryWithCount() throws Exception{
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/54931a52/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
----------------------------------------------------------------------
diff --git a/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml b/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
new file mode 100644
index 0000000..8d9fd97
--- /dev/null
+++ b/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<!DOCTYPE cache PUBLIC
+ "-//GemStone Systems, Inc.//GemFire Declarative Caching 8.0//EN"
+ "http://www.gemstone.com/dtd/cache8_0.dtd">
+<cache>
+<disk-store name="diskStore">
+ <disk-dirs>
+ <disk-dir>diskDir-PersistentPartitionWithIndexDiskStore</disk-dir>
+ </disk-dirs>
+ </disk-store>
+
+<region name="persistentTestRegion">
+<region-attributes refid="PARTITION_REDUNDANT_PERSISTENT" disk-store-name="diskStore" >
+</region-attributes>
+<index name="persistentTestRegion_ID" from-clause="/persistentTestRegion" expression="ID" type="range"/>
+</region>
+</cache>
[10/14] incubator-geode git commit: GEODE-249: Added license header
to test
Posted by kl...@apache.org.
GEODE-249: Added license header to test
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/0d86c3bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/0d86c3bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/0d86c3bd
Branch: refs/heads/feature/GEODE-1050
Commit: 0d86c3bd74cdfa75f6dc3f54bf523e41e2df4984
Parents: aab357f
Author: Jason Huynh <hu...@gmail.com>
Authored: Fri Mar 18 13:50:24 2016 -0700
Committer: Jason Huynh <hu...@gmail.com>
Committed: Fri Mar 18 13:50:24 2016 -0700
----------------------------------------------------------------------
.../cache/PartitionedRegionQueryEvaluatorTest.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/0d86c3bd/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
index ac9be94..fa4a26a 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.gemstone.gemfire.internal.cache;
import static org.junit.Assert.assertEquals;
[02/14] incubator-geode git commit: GEODE-249: Retry on local node
does not overwrite previously retrieved results
Posted by kl...@apache.org.
GEODE-249: Retry on local node does not overwrite previously retrieved results
If a query on a remote node fails and buckets are determined to have moved to
the local node, a retry of the query is executed on the local node. However
the code currently will overwrite any results the local node had already
correctly retrieved.
Added new junit test for PartitionedRegionQueryEvaluator and renamed existing
test to PartitionedRegionQueryEvaluatorIntegrationTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/aab357f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/aab357f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/aab357f6
Branch: refs/heads/feature/GEODE-1050
Commit: aab357f6925f88324987e2246879faeb33ea7962
Parents: 54931a5
Author: Jason Huynh <hu...@gmail.com>
Authored: Fri Mar 11 15:27:33 2016 -0800
Committer: Jason Huynh <hu...@gmail.com>
Committed: Fri Mar 18 13:35:03 2016 -0700
----------------------------------------------------------------------
.../gemfire/internal/cache/AbstractRegion.java | 2 +-
.../cache/PartitionedRegionDataStore.java | 13 +-
.../cache/PartitionedRegionQueryEvaluator.java | 40 +-
...onedRegionQueryEvaluatorIntegrationTest.java | 307 +++++++++++++++
...artitionedRegionQueryEvaluatorJUnitTest.java | 307 ---------------
.../PartitionedRegionQueryEvaluatorTest.java | 373 +++++++++++++++++++
6 files changed, 725 insertions(+), 317 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/aab357f6/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java
index 4711f80..9a55195 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java
@@ -1969,7 +1969,7 @@ public abstract class AbstractRegion implements Region, RegionAttributes,
return this.serialNumber;
}
- public final GemFireCacheImpl getCache() {
+ public /*final*/ GemFireCacheImpl getCache() {
return this.cache;
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/aab357f6/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java
index 5f28c93..010812d 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java
@@ -183,6 +183,17 @@ public class PartitionedRegionDataStore implements HasCachePerfStats
private static final boolean UPDATE_ACCESS_TIME_ON_INTEREST = Boolean
.getBoolean("gemfire.updateAccessTimeOnClientInterest");
+
+ //Only for testing
+ PartitionedRegionDataStore() {
+ this.bucketCreationLock = null;
+ bucketStats = null;
+ partitionedRegion = null;
+ maximumLocalBytes = -1;
+ this.localBucket2RegionMap = new ConcurrentHashMap<Integer, BucketRegion>();
+ keysOfInterest = null;
+ }
+
/**
* Creates PartitionedRegionDataStore for dataStorage of PR and starts a
* PartitionService to handle remote operations on this DataStore from other
@@ -261,7 +272,7 @@ public class PartitionedRegionDataStore implements HasCachePerfStats
* the id of the bucket
* @return true if the provided bucket is being managed
*/
- public final boolean isManagingBucket(int bucketId)
+ public boolean isManagingBucket(int bucketId)
{
BucketRegion buk = this.localBucket2RegionMap.get(Integer.valueOf(bucketId));
if (buk != null && !buk.isDestroyed()) {
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/aab357f6/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluator.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluator.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluator.java
index 1d6cf0e..9dfa94c 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluator.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluator.java
@@ -322,13 +322,13 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
} else {
// send separate message to each recipient since each one has a
// different list of bucket ids
- processor = new StreamingQueryPartitionResponse(this.sys, n2b.keySet());
+ processor = createStreamingQueryPartitionResponse(this.sys, n2b);
for (Iterator<Map.Entry<InternalDistributedMember,List<Integer>>> itr = n2b.entrySet().iterator(); itr.hasNext();) {
Map.Entry<InternalDistributedMember , List<Integer>> me = itr.next();
final InternalDistributedMember rcp = me.getKey();
final List<Integer> bucketIds = me.getValue();
DistributionMessage m = createRequestMessage(rcp, processor, bucketIds);
- Set notReceivedMembers = this.sys.getDistributionManager().putOutgoing(m);
+ Set notReceivedMembers = sendMessage(m);
if (th != null) {
th.hook(4);
}
@@ -425,6 +425,14 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
return requiresRetry | localNeedsRetry;
}
+ protected Set sendMessage(DistributionMessage m) {
+ return this.sys.getDistributionManager().putOutgoing(m);
+ }
+
+ protected StreamingQueryPartitionResponse createStreamingQueryPartitionResponse(InternalDistributedSystem system,HashMap<InternalDistributedMember, List<Integer>> n2b) {
+ return new StreamingQueryPartitionResponse(system, n2b.keySet());
+ }
+
/**
* Executes a query over the provided buckets in a <code>PartitionedRegion</code>.
@@ -820,7 +828,7 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
* @return the node-to-bucket map
*/
- // (package access, and returns map for unit test purposes)
+ // (package access for unit test purposes)
Map<InternalDistributedMember, List<Integer>> buildNodeToBucketMap() throws QueryException
{
return buildNodeToBucketMapForBuckets(this.bucketsToQuery);
@@ -858,7 +866,7 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
}
}
- final List allNodes = new ArrayList(this.pr.getRegionAdvisor().adviseDataStore());
+ final List allNodes = getAllNodes();
/*
for(Map.Entry<InternalDistributedMember, Collection<Collection>> entry : resultsPerMember.entrySet()) {
InternalDistributedMember member = entry.getKey();
@@ -880,7 +888,7 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
final List<Integer> buckets = new ArrayList<Integer>();
for (Integer bid : bucketIdsToConsider) {
if (!bucketIds.contains(bid)) {
- final Set owners = pr.getRegionAdvisor().getBucketOwners(bid.intValue());
+ final Set owners = getBucketOwners(bid);
if (owners.contains(nd)) {
buckets.add(bid);
bucketIds.add(bid);
@@ -904,6 +912,14 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
return ret;
}
+ protected Set<InternalDistributedMember> getBucketOwners(Integer bid) {
+ return pr.getRegionAdvisor().getBucketOwners(bid.intValue());
+ }
+
+ protected ArrayList getAllNodes() {
+ return new ArrayList(this.pr.getRegionAdvisor().adviseDataStore());
+ }
+
/**
* Executes query on local data store.
*
@@ -926,7 +942,7 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
List<Integer> bucketList = this.node2bucketIds.get(me);
try {
- PRQueryProcessor qp = new PRQueryProcessor(this.pr, query, parameters, bucketList);
+ PRQueryProcessor qp = createLocalPRQueryProcessor(bucketList);
MemberResultsList resultCollector = new MemberResultsList();
// Execute Query.
@@ -975,8 +991,12 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
}
resultCollector.setLastChunkReceived(true);
- // Add results to the results-list.
- this.resultsPerMember.put(me, resultCollector);
+ // Add results to the results-list. If prior successfully completed
+ //results exist from previous executions on different buckets, add (to) those results as well.
+ MemberResultsList otherResults = (MemberResultsList)this.resultsPerMember.put(me, resultCollector);
+ if (otherResults != null) {
+ resultCollector.addAll(otherResults);
+ }
} catch (ForceReattemptException retryRequired) {
if (logger.isDebugEnabled()) {
@@ -988,6 +1008,10 @@ public class PartitionedRegionQueryEvaluator extends StreamingPartitionOperation
return false;
}
+ protected PRQueryProcessor createLocalPRQueryProcessor(List<Integer> bucketList) {
+ return new PRQueryProcessor(this.pr, query, parameters, bucketList);
+ }
+
protected void memberStreamCorrupted(InternalDistributedMember sender) {
this.resultsPerMember.remove(sender);
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/aab357f6/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorIntegrationTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorIntegrationTest.java
new file mode 100644
index 0000000..8f9b392
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorIntegrationTest.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.internal.cache;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.gemstone.gemfire.LogWriter;
+import com.gemstone.gemfire.cache.Cache;
+import com.gemstone.gemfire.cache.Region;
+import com.gemstone.gemfire.cache.Scope;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.internal.cache.BucketAdvisor.BucketProfile;
+import com.gemstone.gemfire.internal.cache.partitioned.RegionAdvisor;
+import com.gemstone.gemfire.internal.cache.partitioned.RegionAdvisor.PartitionProfile;
+import com.gemstone.gemfire.internal.util.VersionedArrayList;
+import com.gemstone.gemfire.test.junit.categories.IntegrationTest;
+
+/**
+ * This class is an integration test for <code>PartitionedRegionQueryEvaluator</code> class.
+ * @author rreja
+ */
+@Category(IntegrationTest.class)
+public class PartitionedRegionQueryEvaluatorIntegrationTest
+{
+ @Rule public TestName name = new TestName();
+ LogWriter logger = null;
+
+ @Before
+ public void setUp() throws Exception
+ {
+ if (logger == null) {
+ logger = PartitionedRegionTestHelper.getLogger();
+ }
+ }
+
+ /**
+ * Test for the helper method getNodeToBucketMap.
+ *
+ */
+ @Test
+ public void testGetNodeToBucketMap()
+ {
+ int totalNodes = 100;
+ String prPrefix = name.getMethodName();
+ String localMaxMemory = "0";
+ final int redundancy = 1;
+ final int totalNoOfBuckets = 5;
+ PartitionedRegion pr = (PartitionedRegion)PartitionedRegionTestHelper
+ .createPartitionedRegion(prPrefix, localMaxMemory, redundancy);
+
+ HashSet<Integer> bucketsToQuery = new HashSet<Integer>();
+ for (int i = 0; i < totalNoOfBuckets; i++) {
+ bucketsToQuery.add(i);
+ }
+ final String expectedUnknownHostException = UnknownHostException.class
+ .getName();
+ pr.getCache().getLogger().info(
+ "<ExpectedException action=add>" + expectedUnknownHostException
+ + "</ExpectedException>");
+ final ArrayList nodes = createNodeList(totalNodes);
+ pr.getCache().getLogger().info(
+ "<ExpectedException action=remove>" + expectedUnknownHostException
+ + "</ExpectedException>");
+ // populating bucket2Node of the partition region
+ // ArrayList<InternalDistributedMember>
+ final ArrayList dses = createDataStoreList(totalNodes);
+ populateBucket2Node(pr, dses, totalNoOfBuckets);
+
+ populateAllPartitionedRegion(pr, nodes);
+
+ // running the algorithm and getting the list of bucktes to grab
+ PartitionedRegionQueryEvaluator evalr = new PartitionedRegionQueryEvaluator(pr.getSystem(), pr, null, null, null, bucketsToQuery);
+ Map n2bMap = null;
+ try {
+ n2bMap = evalr.buildNodeToBucketMap();
+ } catch (Exception ex) {
+
+ }
+ ArrayList buckList = new ArrayList();
+ for (Iterator itr = n2bMap.entrySet().iterator(); itr.hasNext();) {
+ Map.Entry entry = (Map.Entry)itr.next();
+ if (entry.getValue() != null)
+ buckList.addAll((List)entry.getValue());
+ }
+ // checking size of the two lists
+ assertEquals("Unexpected number of buckets", totalNoOfBuckets, buckList.size());
+ for (int i = 0; i < totalNoOfBuckets; i++) {
+ assertTrue(" Bucket with Id = " + i + " not present in bucketList.",
+ buckList.contains(new Integer(i)));
+ }
+ clearAllPartitionedRegion(pr);
+ logger.info("************test ended successfully **********");
+
+ }
+
+ /**
+ * This function populates bucket2Node region of the partition region
+ *
+ * @param pr
+ */
+ public void populateBucket2Node(PartitionedRegion pr, List nodes,
+ int numOfBuckets)
+ {
+ assertEquals(0, pr.getRegionAdvisor().getCreatedBucketsCount());
+ final RegionAdvisor ra = pr.getRegionAdvisor();
+ int nodeListCnt = 0;
+ Random ran = new Random();
+ HashMap verMap = new HashMap(); // Map tracking version for profile insertion purposes
+ for (int i = 0; i < numOfBuckets; i++) {
+ nodeListCnt = setNodeListCnt(nodeListCnt);
+ for (int j = 0; j < nodeListCnt; j++) {
+ BucketProfile bp = new BucketProfile();
+ bp.peerMemberId = (InternalDistributedMember)
+ nodes.get(ran.nextInt(nodes.size()));
+ Integer v;
+ if ((v = (Integer) verMap.get(bp.getDistributedMember())) != null) {
+ bp.version = v.intValue() + 1;
+ verMap.put(bp.getDistributedMember(), new Integer(bp.version));
+ } else {
+ verMap.put(bp.getDistributedMember(), new Integer(0));
+ bp.version = 0;
+ }
+
+ bp.isHosting = true;
+ if (j == 0) {
+ bp.isPrimary = true;
+ }
+ bp.scope = Scope.DISTRIBUTED_ACK;
+ boolean forceBadProfile = true;
+ assertTrue(ra.getBucket(i).getBucketAdvisor().putProfile(bp, forceBadProfile));
+ }
+ }
+ }
+
+ private void clearAllPartitionedRegion(PartitionedRegion pr) {
+ Cache cache = pr.getCache();
+ Region allPR = PartitionedRegionHelper.getPRRoot(cache);
+ allPR.clear();
+ }
+
+ /**
+ * This function decides number of the nodes in the list of bucket2Node region
+ *
+ * @param i
+ * @return
+ */
+ private int setNodeListCnt(int i)
+ {
+ int nListcnt = 0;
+ switch (i) {
+ case 0:
+ nListcnt = 1;
+ break;
+ case 1:
+ nListcnt = 4;
+ break;
+ case 2:
+ nListcnt = 1;
+ break;
+ case 3:
+ nListcnt = 2;
+ break;
+ case 4:
+ nListcnt = 1;
+ break;
+ case 5:
+ nListcnt = 3;
+ break;
+ case 6:
+ nListcnt = 3;
+ break;
+ case 7:
+ nListcnt = 1;
+ break;
+ case 8:
+ nListcnt = 1;
+ break;
+ case 9:
+ nListcnt = 2;
+ break;
+
+ }
+ return nListcnt;
+ }
+
+ /**
+ * This functions number of new nodes specified by nCount.
+ *
+ * @param nCount
+ * @return
+ */
+ private ArrayList createNodeList(int nCount)
+ {
+ ArrayList nodeList = new ArrayList(nCount);
+ for (int i = 0; i < nCount; i++) {
+ nodeList.add(createNode(i));
+ }
+ return nodeList;
+ }
+
+ private ArrayList createDataStoreList(int nCount)
+ {
+ // ArrayList<InternalDistributedMember>
+ ArrayList nodeList = new ArrayList(nCount);
+ for (int i = 0; i < nCount; i++) {
+ nodeList.add(createDataStoreMember(i));
+ }
+ return nodeList;
+ }
+
+ private VersionedArrayList getVersionedNodeList(int nCount, List<Node> nodes)
+ {
+ VersionedArrayList nodeList = new VersionedArrayList(nCount);
+ Random ran = new Random();
+ for (int i = 0; i < nCount; i++) {
+ nodeList.add(nodes.get(ran.nextInt(nodes.size())));
+ }
+ return nodeList;
+ }
+
+ private InternalDistributedMember createDataStoreMember(int i)
+ {
+ String hostname = null;
+ InternalDistributedMember mem = null;
+ try {
+ mem = new InternalDistributedMember("host" + i, 3033);
+ }
+ catch (java.net.UnknownHostException uhe) {
+ logger.severe("PartitionedRegion: initalizeNode() Unknown host = "
+ + hostname + " servicePort = " + 0, uhe);
+ throw new PartitionedRegionException(
+ "PartitionedRegionDataStore: initalizeNode() Unknown host = "
+ + hostname + " servicePort = " + 0, uhe);
+ }
+ return mem;
+ }
+
+ /**
+ * this function creates new node.
+ *
+ * @return
+ */
+ public Node createNode(int i)
+ {
+ Node node = null;
+ try {
+ node = new Node(new InternalDistributedMember("host" + i, 3033), i);
+ node.setPRType(Node.DATASTORE);
+ }
+ catch (java.net.UnknownHostException uhe) {
+ logger.severe("PartitionedRegion: initalizeNode() threw exception", uhe);
+ throw new PartitionedRegionException("", uhe);
+ }
+ return node;
+ }
+
+ private void populateAllPartitionedRegion(PartitionedRegion pr, List nodes)
+ {
+ // int totalNodes = 4;
+ Region rootReg = PartitionedRegionHelper.getPRRoot(pr.getCache());
+// Region allPRs = PartitionedRegionHelper.getPRConfigRegion(rootReg, pr
+// .getCache());
+ PartitionRegionConfig prConf = new PartitionRegionConfig(pr.getPRId(), pr
+ .getFullPath(), pr.getPartitionAttributes(), pr.getScope());
+ RegionAdvisor ra = pr.getRegionAdvisor();
+ for (Iterator itr = nodes.iterator(); itr.hasNext();) {
+ Node n = (Node)itr.next();
+ prConf.addNode(n);
+ PartitionProfile pp = (PartitionProfile) ra.createProfile();
+ pp.peerMemberId = n.getMemberId();
+ pp.isDataStore = true;
+ final boolean forceFakeProfile = true;
+ pr.getRegionAdvisor().putProfile(pp, forceFakeProfile);
+ }
+ rootReg.put(pr.getRegionIdentifier(), prConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/aab357f6/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorJUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorJUnitTest.java
deleted file mode 100644
index 2d30073..0000000
--- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorJUnitTest.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.gemstone.gemfire.internal.cache;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-
-import com.gemstone.gemfire.LogWriter;
-import com.gemstone.gemfire.cache.Cache;
-import com.gemstone.gemfire.cache.Region;
-import com.gemstone.gemfire.cache.Scope;
-import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
-import com.gemstone.gemfire.internal.cache.BucketAdvisor.BucketProfile;
-import com.gemstone.gemfire.internal.cache.partitioned.RegionAdvisor;
-import com.gemstone.gemfire.internal.cache.partitioned.RegionAdvisor.PartitionProfile;
-import com.gemstone.gemfire.internal.util.VersionedArrayList;
-import com.gemstone.gemfire.test.junit.categories.IntegrationTest;
-
-/**
- * This class is junit test for <code>PartitionedRegionQueryEvaluator</code> class.
- * @author rreja
- */
-@Category(IntegrationTest.class)
-public class PartitionedRegionQueryEvaluatorJUnitTest
-{
- @Rule public TestName name = new TestName();
- LogWriter logger = null;
-
- @Before
- public void setUp() throws Exception
- {
- if (logger == null) {
- logger = PartitionedRegionTestHelper.getLogger();
- }
- }
-
- /**
- * Test for the helper method getNodeToBucketMap.
- *
- */
- @Test
- public void testGetNodeToBucketMap()
- {
- int totalNodes = 100;
- String prPrefix = name.getMethodName();
- String localMaxMemory = "0";
- final int redundancy = 1;
- final int totalNoOfBuckets = 5;
- PartitionedRegion pr = (PartitionedRegion)PartitionedRegionTestHelper
- .createPartitionedRegion(prPrefix, localMaxMemory, redundancy);
-
- HashSet<Integer> bucketsToQuery = new HashSet<Integer>();
- for (int i = 0; i < totalNoOfBuckets; i++) {
- bucketsToQuery.add(i);
- }
- final String expectedUnknownHostException = UnknownHostException.class
- .getName();
- pr.getCache().getLogger().info(
- "<ExpectedException action=add>" + expectedUnknownHostException
- + "</ExpectedException>");
- final ArrayList nodes = createNodeList(totalNodes);
- pr.getCache().getLogger().info(
- "<ExpectedException action=remove>" + expectedUnknownHostException
- + "</ExpectedException>");
- // populating bucket2Node of the partition region
- // ArrayList<InternalDistributedMember>
- final ArrayList dses = createDataStoreList(totalNodes);
- populateBucket2Node(pr, dses, totalNoOfBuckets);
-
- populateAllPartitionedRegion(pr, nodes);
-
- // running the algorithm and getting the list of bucktes to grab
- PartitionedRegionQueryEvaluator evalr = new PartitionedRegionQueryEvaluator(pr.getSystem(), pr, null, null, null, bucketsToQuery);
- Map n2bMap = null;
- try {
- n2bMap = evalr.buildNodeToBucketMap();
- } catch (Exception ex) {
-
- }
- ArrayList buckList = new ArrayList();
- for (Iterator itr = n2bMap.entrySet().iterator(); itr.hasNext();) {
- Map.Entry entry = (Map.Entry)itr.next();
- if (entry.getValue() != null)
- buckList.addAll((List)entry.getValue());
- }
- // checking size of the two lists
- assertEquals("Unexpected number of buckets", totalNoOfBuckets, buckList.size());
- for (int i = 0; i < totalNoOfBuckets; i++) {
- assertTrue(" Bucket with Id = " + i + " not present in bucketList.",
- buckList.contains(new Integer(i)));
- }
- clearAllPartitionedRegion(pr);
- logger.info("************test ended successfully **********");
-
- }
-
- /**
- * This function populates bucket2Node region of the partition region
- *
- * @param pr
- */
- public void populateBucket2Node(PartitionedRegion pr, List nodes,
- int numOfBuckets)
- {
- assertEquals(0, pr.getRegionAdvisor().getCreatedBucketsCount());
- final RegionAdvisor ra = pr.getRegionAdvisor();
- int nodeListCnt = 0;
- Random ran = new Random();
- HashMap verMap = new HashMap(); // Map tracking version for profile insertion purposes
- for (int i = 0; i < numOfBuckets; i++) {
- nodeListCnt = setNodeListCnt(nodeListCnt);
- for (int j = 0; j < nodeListCnt; j++) {
- BucketProfile bp = new BucketProfile();
- bp.peerMemberId = (InternalDistributedMember)
- nodes.get(ran.nextInt(nodes.size()));
- Integer v;
- if ((v = (Integer) verMap.get(bp.getDistributedMember())) != null) {
- bp.version = v.intValue() + 1;
- verMap.put(bp.getDistributedMember(), new Integer(bp.version));
- } else {
- verMap.put(bp.getDistributedMember(), new Integer(0));
- bp.version = 0;
- }
-
- bp.isHosting = true;
- if (j == 0) {
- bp.isPrimary = true;
- }
- bp.scope = Scope.DISTRIBUTED_ACK;
- boolean forceBadProfile = true;
- assertTrue(ra.getBucket(i).getBucketAdvisor().putProfile(bp, forceBadProfile));
- }
- }
- }
-
- private void clearAllPartitionedRegion(PartitionedRegion pr) {
- Cache cache = pr.getCache();
- Region allPR = PartitionedRegionHelper.getPRRoot(cache);
- allPR.clear();
- }
-
- /**
- * This function decides number of the nodes in the list of bucket2Node region
- *
- * @param i
- * @return
- */
- private int setNodeListCnt(int i)
- {
- int nListcnt = 0;
- switch (i) {
- case 0:
- nListcnt = 1;
- break;
- case 1:
- nListcnt = 4;
- break;
- case 2:
- nListcnt = 1;
- break;
- case 3:
- nListcnt = 2;
- break;
- case 4:
- nListcnt = 1;
- break;
- case 5:
- nListcnt = 3;
- break;
- case 6:
- nListcnt = 3;
- break;
- case 7:
- nListcnt = 1;
- break;
- case 8:
- nListcnt = 1;
- break;
- case 9:
- nListcnt = 2;
- break;
-
- }
- return nListcnt;
- }
-
- /**
- * This functions number of new nodes specified by nCount.
- *
- * @param nCount
- * @return
- */
- private ArrayList createNodeList(int nCount)
- {
- ArrayList nodeList = new ArrayList(nCount);
- for (int i = 0; i < nCount; i++) {
- nodeList.add(createNode(i));
- }
- return nodeList;
- }
-
- private ArrayList createDataStoreList(int nCount)
- {
- // ArrayList<InternalDistributedMember>
- ArrayList nodeList = new ArrayList(nCount);
- for (int i = 0; i < nCount; i++) {
- nodeList.add(createDataStoreMember(i));
- }
- return nodeList;
- }
-
- private VersionedArrayList getVersionedNodeList(int nCount, List<Node> nodes)
- {
- VersionedArrayList nodeList = new VersionedArrayList(nCount);
- Random ran = new Random();
- for (int i = 0; i < nCount; i++) {
- nodeList.add(nodes.get(ran.nextInt(nodes.size())));
- }
- return nodeList;
- }
-
- private InternalDistributedMember createDataStoreMember(int i)
- {
- String hostname = null;
- InternalDistributedMember mem = null;
- try {
- mem = new InternalDistributedMember("host" + i, 3033);
- }
- catch (java.net.UnknownHostException uhe) {
- logger.severe("PartitionedRegion: initalizeNode() Unknown host = "
- + hostname + " servicePort = " + 0, uhe);
- throw new PartitionedRegionException(
- "PartitionedRegionDataStore: initalizeNode() Unknown host = "
- + hostname + " servicePort = " + 0, uhe);
- }
- return mem;
- }
-
- /**
- * this function creates new node.
- *
- * @return
- */
- public Node createNode(int i)
- {
- Node node = null;
- try {
- node = new Node(new InternalDistributedMember("host" + i, 3033), i);
- node.setPRType(Node.DATASTORE);
- }
- catch (java.net.UnknownHostException uhe) {
- logger.severe("PartitionedRegion: initalizeNode() threw exception", uhe);
- throw new PartitionedRegionException("", uhe);
- }
- return node;
- }
-
- private void populateAllPartitionedRegion(PartitionedRegion pr, List nodes)
- {
- // int totalNodes = 4;
- Region rootReg = PartitionedRegionHelper.getPRRoot(pr.getCache());
-// Region allPRs = PartitionedRegionHelper.getPRConfigRegion(rootReg, pr
-// .getCache());
- PartitionRegionConfig prConf = new PartitionRegionConfig(pr.getPRId(), pr
- .getFullPath(), pr.getPartitionAttributes(), pr.getScope());
- RegionAdvisor ra = pr.getRegionAdvisor();
- for (Iterator itr = nodes.iterator(); itr.hasNext();) {
- Node n = (Node)itr.next();
- prConf.addNode(n);
- PartitionProfile pp = (PartitionProfile) ra.createProfile();
- pp.peerMemberId = n.getMemberId();
- pp.isDataStore = true;
- final boolean forceFakeProfile = true;
- pr.getRegionAdvisor().putProfile(pp, forceFakeProfile);
- }
- rootReg.put(pr.getRegionIdentifier(), prConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/aab357f6/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
new file mode 100644
index 0000000..ac9be94
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/PartitionedRegionQueryEvaluatorTest.java
@@ -0,0 +1,373 @@
+package com.gemstone.gemfire.internal.cache;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.gemstone.gemfire.cache.CacheException;
+import com.gemstone.gemfire.cache.query.QueryException;
+import com.gemstone.gemfire.cache.query.SelectResults;
+import com.gemstone.gemfire.cache.query.internal.CompiledSelect;
+import com.gemstone.gemfire.cache.query.internal.CompiledValue;
+import com.gemstone.gemfire.cache.query.internal.DefaultQuery;
+import com.gemstone.gemfire.cache.query.internal.LinkedResultSet;
+import com.gemstone.gemfire.cache.query.internal.types.ObjectTypeImpl;
+import com.gemstone.gemfire.distributed.internal.DistributionMessage;
+import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.test.fake.Fakes;
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+
+@Category(UnitTest.class)
+public class PartitionedRegionQueryEvaluatorTest {
+
+ InternalDistributedMember localNode;
+ InternalDistributedMember remoteNodeA;
+ InternalDistributedMember remoteNodeB;
+ GemFireCacheImpl cache;
+ InternalDistributedSystem system;
+ PartitionedRegion pr;
+ DefaultQuery query;
+ //Needed to help mock out certain scenarios
+ ExtendedPartitionedRegionDataStore dataStore;
+ //This is the set of nodes that remain after a failure
+ ArrayList allNodes = new ArrayList();
+ //convenience list for empty set
+ Set<InternalDistributedMember> noFailingMembers = new HashSet<>();
+
+ @Before
+ public void setup() throws Exception {
+ localNode = new InternalDistributedMember("localhost",8888);
+ remoteNodeA = new InternalDistributedMember("localhost",8889);
+ remoteNodeB = new InternalDistributedMember("localhost",8890);
+ cache = Fakes.cache();
+ system = (InternalDistributedSystem) cache.getDistributedSystem();
+
+ allNodes.add(localNode);
+ allNodes.add(remoteNodeA);
+ allNodes.add(remoteNodeB);
+
+ pr = mock(PartitionedRegion.class);
+ dataStore = new ExtendedPartitionedRegionDataStore();
+ CompiledSelect select = mock(CompiledSelect.class);
+ when(select.getType()).thenReturn(CompiledValue.COMPARISON);
+ when(select.getElementTypeForOrderByQueries()).thenReturn(new ObjectTypeImpl(String.class));
+ query = mock(DefaultQuery.class);
+ when(query.getSimpleSelect()).thenReturn(select);
+ when(query.getLimit(any())).thenReturn(-1);
+ when (pr.getCachePerfStats()).thenReturn(mock(CachePerfStats.class));
+ when(pr.getMyId()).thenReturn(localNode);
+ when(pr.getDataStore()).thenReturn(dataStore);
+ when(pr.getCache()).thenReturn(cache);
+
+ }
+
+ @Test
+ public void testLocalQueryReturnsResultsToPartitionedQueryEvaluator() throws Exception {
+ List resultsForMember1 = createResultObjects("1", "2", "3");
+
+ PartitionedQueryScenario scenario = new PartitionedQueryScenario(localNode, allNodes, noFailingMembers, createFakeBucketMap(), new ProcessDataFaker() {
+ public void processData(PartitionedRegionQueryEvaluator prqe) {
+ //this test won't have any remote nodes responding
+ }
+
+ public void executeQueryLocally(Collection resultsCollector) {
+ resultsCollector.add(resultsForMember1);
+ }
+ });
+
+ Set allBucketsToQuery = scenario.getAllBucketsToQuery();
+ Queue<PartitionedQueryScenario> scenarios = createScenariosQueue(scenario);
+ dataStore.setScenarios(scenarios);
+
+ PartitionedRegionQueryEvaluator prqe = new ExtendedPartitionedRegionQueryEvaluator(system, pr, query, null, new LinkedResultSet(), allBucketsToQuery, scenarios);
+ Collection results = prqe.queryBuckets(null).asList();
+ assertNotNull(results);
+ assertEquals(resultsForMember1.size(), results.size());
+ results.removeAll(resultsForMember1);
+ assertTrue(results.isEmpty());
+ }
+
+ @Test
+ public void testRemoteAndLocalQueryReturnsResultsToPartitionedQueryEvaluator() throws Exception {
+ List resultsForMember1 = createResultObjects("1", "2", "3");
+ List resultsForMember2 = createResultObjects("4", "5", "6");
+
+ PartitionedQueryScenario scenario = new PartitionedQueryScenario(localNode, allNodes, noFailingMembers, createFakeBucketMap(), new ProcessDataFaker() {
+ public void processData(PartitionedRegionQueryEvaluator prqe) {
+ prqe.processData(resultsForMember2, remoteNodeA, 0, true);
+ }
+
+ public void executeQueryLocally(Collection resultsCollector) {
+ resultsCollector.add(resultsForMember1);
+ }
+ });
+
+ Set allBucketsToQuery = scenario.getAllBucketsToQuery();
+ Queue<PartitionedQueryScenario> scenarios = createScenariosQueue(scenario);
+ dataStore.setScenarios(scenarios);
+
+ PartitionedRegionQueryEvaluator prqe = new ExtendedPartitionedRegionQueryEvaluator(system, pr, query, null, new LinkedResultSet(), allBucketsToQuery, scenarios);
+ Collection results = prqe.queryBuckets(null).asList();
+ List expectedResults = new LinkedList();
+ expectedResults.addAll(resultsForMember1);
+ expectedResults.addAll(resultsForMember2);
+ assertNotNull(results);
+ assertEquals(expectedResults.size(), results.size());
+ results.removeAll(expectedResults);
+ assertTrue(results.isEmpty());
+ }
+
+
+ @Test
+ public void testFailingRemoteNodeAndRetryOnLocalNodeDoesNotSquashResultsOfOriginalQueryOnLocalNode() throws Exception {
+ List resultsForMember1 = createResultObjects("1", "2", "3");
+ List resultsForMember2 = createResultObjects("A", "B", "C");
+ List resultsForMember1ForRetry = createResultObjects("&", "$", "!");
+
+ Set<InternalDistributedMember> failingMembers = new HashSet<>();
+ failingMembers.add(remoteNodeB);
+ PartitionedQueryScenario allNodesUpAtBeginning = new PartitionedQueryScenario(localNode, allNodes, failingMembers, createFakeBucketMap(), new ProcessDataFaker() {
+ public void processData(PartitionedRegionQueryEvaluator prqe) {
+ prqe.processData(resultsForMember2, remoteNodeA, 0, true);
+ }
+
+ public void executeQueryLocally(Collection resultsCollector) {
+ resultsCollector.add(resultsForMember1);
+ }
+ });
+
+ PartitionedQueryScenario afterFailureScenario = new PartitionedQueryScenario(localNode, allNodes, noFailingMembers, createFakeBucketMapFailedNodesToLocalMember(), new ProcessDataFaker() {
+ public void processData(PartitionedRegionQueryEvaluator prqe) {
+ //on retry we do not need to fake a retry on a remote node for this test
+ }
+ public void executeQueryLocally(Collection resultsCollector) {
+ resultsCollector.add(resultsForMember1);
+ }
+ });
+ Set allBucketsToQuery = allNodesUpAtBeginning.getAllBucketsToQuery();
+ Queue<PartitionedQueryScenario> scenarios = createScenariosQueue(allNodesUpAtBeginning, afterFailureScenario);
+ dataStore.setScenarios(scenarios);
+
+ PartitionedRegionQueryEvaluator prqe = new ExtendedPartitionedRegionQueryEvaluator(system, pr, query, null, new LinkedResultSet(), allBucketsToQuery, scenarios);
+ Collection results = prqe.queryBuckets(null).asList();
+
+ List expectedResults = new LinkedList();
+ expectedResults.addAll(resultsForMember1);
+ expectedResults.addAll(resultsForMember2);
+ expectedResults.addAll(resultsForMember1ForRetry);
+ assertNotNull(results);
+ assertEquals(expectedResults.size(), results.size());
+ results.removeAll(expectedResults);
+ assertTrue(results.isEmpty());
+
+ }
+
+ private Map<InternalDistributedMember, List<Integer>> createFakeBucketMap() {
+ Map<InternalDistributedMember, List<Integer>> bucketToNodeMap = new HashMap<>();
+ bucketToNodeMap.put(localNode, createBucketList(1, 2, 3));
+ bucketToNodeMap.put(remoteNodeA, createBucketList(4, 5, 6));
+ bucketToNodeMap.put(remoteNodeB, createBucketList(7, 8, 9));
+ return bucketToNodeMap;
+ }
+
+ //fake bucket map to use after we fake a node failure
+ private Map<InternalDistributedMember, List<Integer>> createFakeBucketMapFailedNodesToLocalMember() {
+ Map<InternalDistributedMember, List<Integer>> bucketToNodeMap = new HashMap<>();
+ bucketToNodeMap.put(localNode, createBucketList(1, 2, 3, 7, 8, 9));
+ bucketToNodeMap.put(remoteNodeA, createBucketList(4, 5, 6));
+ return bucketToNodeMap;
+ }
+
+ private List<Integer> createBucketList(int... buckets) {
+ List<Integer> bucketList = new ArrayList<>();
+ for (int i : buckets) {
+ bucketList.add(i);
+ }
+ return bucketList;
+ }
+
+ private List createResultObjects(Object... resultObjects) {
+ List results = new LinkedList();
+ for (Object o: resultObjects) {
+ results.add(o);
+ }
+ return results;
+ }
+
+ class ExtendedPartitionedRegionDataStore extends PartitionedRegionDataStore {
+ //Must be the same referenced queue as that used by the ExtendedPartitionedRegionQueryEvaluator
+ //That way they will be synched to the same scenario;
+ Queue<PartitionedQueryScenario> scenarios;
+
+ void setScenarios(Queue<PartitionedQueryScenario> scenarios) {
+ this.scenarios = scenarios;
+ }
+
+ @Override
+ public boolean isManagingBucket(int bucketId) {
+ return scenarios.peek().isLocalManagingBucket(bucketId);
+ }
+ }
+
+ class ExtendedPartitionedRegionQueryEvaluator extends PartitionedRegionQueryEvaluator {
+ Queue<PartitionedQueryScenario> scenarios;
+ //pass through so we can fake out the executeQuery locally
+ PRQueryProcessor extendedPRQueryProcessor;
+
+ public ExtendedPartitionedRegionQueryEvaluator(InternalDistributedSystem sys, PartitionedRegion pr, DefaultQuery query, Object[] parameters,
+ SelectResults cumulativeResults, Set<Integer> bucketsToQuery, Queue<PartitionedQueryScenario> scenarios) {
+ super(sys, pr, query, parameters, cumulativeResults, bucketsToQuery);
+ this.scenarios = scenarios;
+ extendedPRQueryProcessor = new ExtendedPRQueryProcessor(pr, query, null, new LinkedList(bucketsToQuery));
+ }
+
+ private PartitionedQueryScenario currentScenario() {
+ return scenarios.peek();
+ }
+
+ // (package access for unit test purposes)
+ Map<InternalDistributedMember, List<Integer>> buildNodeToBucketMap() throws QueryException
+ {
+ return currentScenario().bucketMap;
+ }
+
+ @Override
+ protected StreamingQueryPartitionResponse createStreamingQueryPartitionResponse(InternalDistributedSystem system, HashMap<InternalDistributedMember, List<Integer>> n2b) {
+ StreamingQueryPartitionResponse response = new FakeNumFailStreamingQueryPartitionResponse(system, n2b, this, scenarios);
+ return response;
+ }
+
+ @Override
+ protected Set sendMessage(DistributionMessage m) {
+ //Don't need to actually send the message...
+ return null;
+ }
+
+ @Override
+ protected PRQueryProcessor createLocalPRQueryProcessor(List<Integer> bucketList) {
+ return extendedPRQueryProcessor;
+ }
+
+ @Override
+ protected ArrayList getAllNodes() {
+ return currentScenario().allNodes;
+ }
+
+ @Override
+ protected Set<InternalDistributedMember> getBucketOwners(Integer bid) {
+ return currentScenario().getBucketOwners(bid);
+ }
+
+ class ExtendedPRQueryProcessor extends PRQueryProcessor {
+
+ public ExtendedPRQueryProcessor(PartitionedRegion pr, DefaultQuery query, Object[] parameters, List buckets) {
+ super(pr, query, parameters, buckets);
+ }
+
+ public ExtendedPRQueryProcessor(PartitionedRegionDataStore prDS, DefaultQuery query, Object[] parameters, List<Integer> buckets) {
+ super(prDS, query, parameters, buckets);
+ }
+
+ @Override
+ public boolean executeQuery(Collection resultsCollector) {
+ currentScenario().processDataFaker.executeQueryLocally(resultsCollector);
+ return true;
+ }
+ }
+
+ class FakeNumFailStreamingQueryPartitionResponse extends StreamingQueryPartitionResponse {
+ private PartitionedRegionQueryEvaluator processor;
+ Queue<PartitionedQueryScenario> scenarios;
+
+ public FakeNumFailStreamingQueryPartitionResponse(InternalDistributedSystem system, HashMap<InternalDistributedMember, List<Integer>> n2b, PartitionedRegionQueryEvaluator processor, Queue<PartitionedQueryScenario> scenarios) {
+ super(system, n2b.keySet());
+ this.processor = processor;
+ this.scenarios = scenarios;
+ }
+
+ @Override
+ public Set<InternalDistributedMember> waitForCacheOrQueryException() throws CacheException, QueryException {
+ currentScenario().processDataFaker.processData(processor);
+ Set<InternalDistributedMember> returnValue = currentScenario().failingNodes;
+ advanceTheScenario();
+ return returnValue;
+ }
+
+ private void advanceTheScenario() {
+ if (scenarios.isEmpty()) {
+ return;
+ }
+ scenarios.remove();
+ }
+ }
+ }
+
+ interface ProcessDataFaker {
+ void processData(PartitionedRegionQueryEvaluator processor);
+ void executeQueryLocally(Collection resultsCollector);
+ }
+
+ //holds information on how the PRQE is to behave and what responses are "returned"
+ private class PartitionedQueryScenario {
+ private InternalDistributedMember localNode;
+ private ArrayList allNodes;
+ private Set<InternalDistributedMember> failingNodes;
+ private ProcessDataFaker processDataFaker;
+ private Map<InternalDistributedMember, List<Integer>> bucketMap;
+
+ public PartitionedQueryScenario(InternalDistributedMember localNode, ArrayList allNodes, Set<InternalDistributedMember> failingNodes, Map<InternalDistributedMember, List<Integer>> bucketMap, ProcessDataFaker processDataFaker) {
+ this.localNode = localNode;
+ this.allNodes = allNodes;
+ this.failingNodes = failingNodes;
+ this.bucketMap = bucketMap;
+ this.processDataFaker = processDataFaker;
+ }
+
+ public Set getAllBucketsToQuery() {
+ Set allBuckets = new HashSet();
+ bucketMap.values().stream().forEach(list -> allBuckets.addAll(list));
+ return allBuckets;
+ }
+
+ public Set<InternalDistributedMember> getBucketOwners(Integer bucketId) {
+ Set<InternalDistributedMember> owners = new HashSet<>();
+ bucketMap.entrySet().stream().forEach(entrySet -> {
+ if (entrySet.getValue().contains(bucketId)) {
+ owners.add(entrySet.getKey());
+ }
+ });
+ return owners;
+ }
+
+ public boolean isLocalManagingBucket(int bucketId) {
+ return getBucketOwners(bucketId).contains(localNode);
+ }
+ }
+
+ private Queue<PartitionedQueryScenario> createScenariosQueue(PartitionedQueryScenario... scenarios) {
+ Queue<PartitionedQueryScenario> queue = new LinkedList<>();
+ for (PartitionedQueryScenario scenario: scenarios) {
+ queue.add(scenario);
+ }
+ return queue;
+
+ }
+}
[06/14] incubator-geode git commit: GEODE-27: A couple of tools to
dump dependencies
Posted by kl...@apache.org.
GEODE-27: A couple of tools to dump dependencies
Adding a task to dump all jars in the binary distribution
This makes it easy to see what dependencies we actually end up shipping
with the product. This dumps all jars found in the distribution or
embedded within war files in the distribution.
Adding a task to determine if a dependency is used
This task checks to see if a dependency is used in the source code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/5839e4d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/5839e4d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/5839e4d2
Branch: refs/heads/feature/GEODE-1050
Commit: 5839e4d2a741dd0b84ce0ed20588fb5bb2d62565
Parents: 0d86c3b
Author: Dan Smith <up...@apache.org>
Authored: Tue Mar 8 16:23:39 2016 -0800
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
build.gradle | 1 +
geode-assembly/build.gradle | 29 ++++++++++
gradle/dependency-resolution.gradle | 98 ++++++++++++++++++++++++++++++++
3 files changed, 128 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/5839e4d2/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 19f72a0..200dd8a 100755
--- a/build.gradle
+++ b/build.gradle
@@ -74,6 +74,7 @@ if (name == 'geode') {
apply from: "${scriptDir}/utilities.gradle"
apply from: "${scriptDir}/java.gradle"
+apply from: "${scriptDir}/dependency-resolution.gradle"
apply from: "${scriptDir}/test.gradle"
apply from: "${scriptDir}/publish.gradle"
apply from: "${scriptDir}/code-analysis.gradle"
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/5839e4d2/geode-assembly/build.gradle
----------------------------------------------------------------------
diff --git a/geode-assembly/build.gradle b/geode-assembly/build.gradle
index d2a5a65..2213614 100755
--- a/geode-assembly/build.gradle
+++ b/geode-assembly/build.gradle
@@ -363,3 +363,32 @@ integrationTest dependOnInstalledProduct
build.dependsOn installDist
installDist.dependsOn ':extensions/geode-modules-assembly:dist'
+
+/**Print the names of all jar files in a fileTree */
+def printJars(tree) {
+ tree.matching {include("**/*.jar")}.visit{ file ->
+ if(!file.isDirectory()) {
+ println file.name
+ }
+ };
+}
+
+task dumpInstalledJars(dependsOn: installDist) << {
+ description "Dump a list of all of the jars shipped with the binary distribution, for validation purposes"
+
+ FileTree installDir = fileTree(dir: installDist.destinationDir)
+
+ println("Jars in the binary install")
+ println("==========================")
+ printJars(installDir)
+
+ installDir.include("**/*.war").visit{ file ->
+ if(!file.isDirectory()) {
+ FileTree warContents = zipTree(file.file)
+ println ""
+ println file.name
+ println("==========================")
+ printJars(warContents);
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/5839e4d2/gradle/dependency-resolution.gradle
----------------------------------------------------------------------
diff --git a/gradle/dependency-resolution.gradle b/gradle/dependency-resolution.gradle
new file mode 100644
index 0000000..1e2b62b
--- /dev/null
+++ b/gradle/dependency-resolution.gradle
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+subprojects {
+ configurations.all {
+ resolutionStrategy {
+ // failOnVersionConflict()
+
+// exclude module: 'aopalliance'
+ // exclude 'asm'
+ // exclude 'aspectjweaver'
+ // exclude 'cglib'
+
+
+ // Force certain versions of transitive dependencies
+ // force 'asm:asm-all:3.3.1'
+ // force 'asm:asm-all:3.3.1'
+ }
+ }
+
+ //Task to dump all depencies of all projects, in a way
+ //that can be diffed before and after dependency changes
+ task dumpDependencies() << {
+ description "Dump all of the dependencies as a flat, sorted list"
+
+ project.configurations.each{ configuration ->
+ println( project.name + ":" + configuration.name )
+ println( '-------------------')
+ configuration.resolvedConfiguration.resolvedArtifacts.collect{dep -> dep.file.name}.unique().toSorted().each{dep ->
+ println(dep)
+ }
+ println()
+ }
+ }
+
+ //Task to find all of the jars in a compile task
+ //that are unused
+ task findUsage() << {
+ description "Find usages of a jar in the source code"
+
+ String jarName = System.getProperty("jar.name")
+ if(jar == null || jar == "") {
+ println "You must specify a jar name: ./gradlew findUsage -Djar.name=commons-io"
+ return
+ }
+ FileTree sourceFiles = compileJava.source
+ FileCollection jars = compileJava.classpath
+
+ File jar = jars.find{file -> file.name.contains(jarName)}
+
+ FileTree jarContents = zipTree(jar)
+ Set packages = new HashSet()
+ jarContents.visit{file ->
+ if(!file.isDirectory() && !file.path.contains("META-INF")) {
+ packages.add(file.relativePath.parent.toString().replace('/', '.'))
+ }
+ }
+
+ println("Packages")
+ println "========"
+ packages.each { p -> println p }
+
+ println ""
+ println("Matches")
+ println "========"
+ sourceFiles.visit{ file ->
+ if(!file.isDirectory()) {
+ boolean matches = false;
+ file.file.eachLine { line ->
+ def matcher = line =~ /^import (.*)\..*;/
+ if(matcher) {
+ def pack = matcher[0][1]
+// println pack
+ matches |= packages.contains(pack)
+ }
+ }
+
+ if(matches) {
+ println file.relativePath;
+ }
+ }
+ }
+ }
+
+}
[03/14] incubator-geode git commit: GEODE-27: Removing some forced
dependencies
Posted by kl...@apache.org.
GEODE-27: Removing some forced dependencies
The version we are picking for these dependencies seemed arbitrary, I
think we should just let them be controlled by gradle.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/8310fd9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/8310fd9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/8310fd9c
Branch: refs/heads/feature/GEODE-1050
Commit: 8310fd9c08e121c7acce80c5e226e35e23478eb7
Parents: 163046f
Author: Dan Smith <up...@apache.org>
Authored: Tue Mar 15 15:05:23 2016 -0700
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
geode-core/build.gradle | 4 +++-
gradle/dependency-resolution.gradle | 4 ----
2 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8310fd9c/geode-core/build.gradle
----------------------------------------------------------------------
diff --git a/geode-core/build.gradle b/geode-core/build.gradle
index 3f4a671..1d12be3 100755
--- a/geode-core/build.gradle
+++ b/geode-core/build.gradle
@@ -80,7 +80,9 @@ dependencies {
transitive=false
}
- compile 'com.google.guava:guava:' + project.'guava.version'
+ compile ('com.google.guava:guava:' + project.'guava.version') {
+ ext.optional = true
+ }
//jsr305 is included only to prevent javadoc warnings about missing annotations in the guava jar
provided 'com.google.code.findbugs:jsr305:' + project.'jsr305.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8310fd9c/gradle/dependency-resolution.gradle
----------------------------------------------------------------------
diff --git a/gradle/dependency-resolution.gradle b/gradle/dependency-resolution.gradle
index e7f1914..dc247d2 100644
--- a/gradle/dependency-resolution.gradle
+++ b/gradle/dependency-resolution.gradle
@@ -23,16 +23,12 @@ subprojects {
//These are forced to match our old non-transitive configuration
//These force statements should be removed and tested with the automatically
//resolved version
- force 'commons-logging:commons-logging:' + project.'commons-logging.version'
force 'org.springframework:spring-context-support:' + project.'springframework.version'
force 'org.springframework:spring-context-support:' + project.'springframework.version'
force 'org.springframework:spring-context:' + project.'springframework.version'
force 'org.springframework:spring-web:' + project.'springframework.version'
force 'org.springframework:spring-tx:' + project.'springframework.version'
force 'org.springframework:spring-webmvc:' + project.'springframework.version'
- force 'javax.activation:activation:' + project.'activation.version'
- force 'commons-beanutils:commons-beanutils:' + project.'commons-beanutils.version'
- force 'com.thoughtworks.paranamer:paranamer:' + project.'paranamer.version'
force 'jline:jline:' + project.'jline.version'
}
}
[12/14] incubator-geode git commit: Merge remote-tracking branch
'origin/develop' into feature/GEODE-1050
Posted by kl...@apache.org.
Merge remote-tracking branch 'origin/develop' into feature/GEODE-1050
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/def98e35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/def98e35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/def98e35
Branch: refs/heads/feature/GEODE-1050
Commit: def98e350f1250e472409f335cd9e227ad4d31fc
Parents: 9c8f1cd 09cd330
Author: Kirk Lund <kl...@apache.org>
Authored: Fri Mar 18 14:34:51 2016 -0700
Committer: Kirk Lund <kl...@apache.org>
Committed: Fri Mar 18 14:34:51 2016 -0700
----------------------------------------------------------------------
build.gradle | 1 +
extensions/geode-modules-hibernate/build.gradle | 15 +-
extensions/geode-modules-tomcat7/build.gradle | 11 +-
extensions/geode-modules/build.gradle | 7 +-
geode-assembly/build.gradle | 51 ++-
.../gemstone/gemfire/BundledJarsJUnitTest.java | 129 ++++++
.../src/test/resources/expected_jars.txt | 82 ++++
geode-core/build.gradle | 125 +++---
.../internal/CumulativeNonDistinctResults.java | 2 -
.../gemfire/internal/cache/AbstractRegion.java | 2 +-
.../internal/cache/AbstractRegionMap.java | 17 +-
.../cache/PartitionedRegionDataStore.java | 13 +-
.../cache/PartitionedRegionQueryEvaluator.java | 40 +-
.../gemfire/cache/query/QueryTestUtils.java | 335 +++-------------
.../cache/query/QueryTestUtilsJUnitTest.java | 31 +-
.../query/dunit/CompactRangeIndexDUnitTest.java | 33 +-
.../cache/query/dunit/HashIndexDUnitTest.java | 29 +-
...itionedRegionCompactRangeIndexDUnitTest.java | 130 +++++++
.../index/CompactRangeIndexJUnitTest.java | 17 +-
...onedRegionQueryEvaluatorIntegrationTest.java | 307 +++++++++++++++
...artitionedRegionQueryEvaluatorJUnitTest.java | 307 ---------------
.../PartitionedRegionQueryEvaluatorTest.java | 389 +++++++++++++++++++
.../dunit/PersistentPartitionWithIndex.xml | 33 ++
geode-junit/build.gradle | 4 +-
geode-lucene/build.gradle | 4 +-
geode-pulse/build.gradle | 25 +-
geode-rebalancer/build.gradle | 9 +-
geode-web-api/build.gradle | 56 ++-
geode-web/build.gradle | 30 +-
gradle/dependency-resolution.gradle | 100 +++++
gradle/java.gradle | 32 --
gradle/publish.gradle | 15 +
32 files changed, 1637 insertions(+), 744 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/def98e35/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/CompactRangeIndexDUnitTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/def98e35/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/HashIndexDUnitTest.java
----------------------------------------------------------------------
[07/14] incubator-geode git commit: GEODE-27: Marking jars as
optional in the generated pom
Posted by kl...@apache.org.
GEODE-27: Marking jars as optional in the generated pom
Adding support to mark jars as optional by setting an ext.optional
property in the dependency declaration.
Adding the optional property to jars in the geode-core project that are
not required for geode to work.
Some of these optional dependencies will go away if we can move things
like the REST API, gfsh, and management classes into another subproject.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/02b95b42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/02b95b42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/02b95b42
Branch: refs/heads/feature/GEODE-1050
Commit: 02b95b42986e3816d436494e7e488db6a9e26565
Parents: dc94a89
Author: Dan Smith <up...@apache.org>
Authored: Thu Mar 10 14:25:12 2016 -0800
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
geode-core/build.gradle | 48 ++++++++++++++++++++++++++++++++++----------
gradle/publish.gradle | 15 ++++++++++++++
2 files changed, 52 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/02b95b42/geode-core/build.gradle
----------------------------------------------------------------------
diff --git a/geode-core/build.gradle b/geode-core/build.gradle
index fe5ab7e..9aa008e 100755
--- a/geode-core/build.gradle
+++ b/geode-core/build.gradle
@@ -41,15 +41,28 @@ dependencies {
exclude module: 'commons-logging-api'
exclude module: 'mx4j-jmx'
exclude module: 'xml-apis'
+ ext.optional = true
+ }
+ compile ('io.netty:netty-all:' + project.'netty-all.version') {
+ ext.optional = true
}
- compile 'io.netty:netty-all:' + project.'netty-all.version'
compile 'it.unimi.dsi:fastutil:' + project.'fastutil.version'
- compile 'javax.mail:javax.mail-api:' + project.'javax.mail-api.version'
+ compile ('javax.mail:javax.mail-api:' + project.'javax.mail-api.version') {
+ ext.optional = true;
+ }
compile 'javax.resource:javax.resource-api:' + project.'javax.resource-api.version'
- compile 'mx4j:mx4j:' + project.'mx4j.version'
- compile 'mx4j:mx4j-remote:' + project.'mx4j.version'
- compile 'mx4j:mx4j-tools:' + project.'mx4j.version'
- compile 'net.java.dev.jna:jna:' + project.'jna.version'
+ compile ('mx4j:mx4j:' + project.'mx4j.version') {
+ ext.optional = true;
+ }
+ compile ('mx4j:mx4j-remote:' + project.'mx4j.version') {
+ ext.optional = true;
+ }
+ compile ('mx4j:mx4j-tools:' + project.'mx4j.version') {
+ ext.optional = true;
+ }
+ compile ('net.java.dev.jna:jna:' + project.'jna.version') {
+ ext.optional = true
+ }
provided ('org.apache.hadoop:hadoop-common:' + project.'hadoop.version') {
transitive=false
}
@@ -71,16 +84,26 @@ dependencies {
compile 'org.apache.logging.log4j:log4j-api:' + project.'log4j.version'
compile 'org.apache.logging.log4j:log4j-core:' + project.'log4j.version'
runtime 'org.fusesource.jansi:jansi:' + project.'jansi.version'
- runtime 'org.apache.logging.log4j:log4j-slf4j-impl:' + project.'log4j.version'
- runtime 'org.apache.logging.log4j:log4j-jcl:' + project.'log4j.version'
- runtime 'org.apache.logging.log4j:log4j-jul:' + project.'log4j.version'
- compile 'org.eclipse.jetty:jetty-webapp:' + project.'jetty.version'
+ runtime ('org.apache.logging.log4j:log4j-slf4j-impl:' + project.'log4j.version') {
+ ext.optional = true
+ }
+ runtime ('org.apache.logging.log4j:log4j-jcl:' + project.'log4j.version') {
+ ext.optional = true
+ }
+ runtime ('org.apache.logging.log4j:log4j-jul:' + project.'log4j.version') {
+ ext.optional = true
+ }
+ compile ('org.eclipse.jetty:jetty-webapp:' + project.'jetty.version') {
+ ext.optional = true
+ }
runtime ('org.springframework:spring-aop:' + project.'springframework.version') {
exclude module: 'aopalliance'
+ ext.optional = true
}
compile ('org.springframework:spring-webmvc:' + project.'springframework.version') {
exclude module: 'aopalliance'
exclude module: 'spring-aop'
+ ext.optional = true
}
compile ('org.springframework.shell:spring-shell:' + project.'spring-shell.version') {
exclude module: 'aopalliance'
@@ -88,8 +111,11 @@ dependencies {
exclude module: 'cglib'
exclude module: 'guava'
exclude module: 'spring-aop'
+ ext.optional = true
+ }
+ compile ('org.xerial.snappy:snappy-java:' + project.'snappy-java.version') {
+ ext.optional = true
}
- compile 'org.xerial.snappy:snappy-java:' + project.'snappy-java.version'
compile project(':geode-common')
compile project(':geode-joptsimple')
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/02b95b42/gradle/publish.gradle
----------------------------------------------------------------------
diff --git a/gradle/publish.gradle b/gradle/publish.gradle
index 73e4cd3..02b0e3c 100644
--- a/gradle/publish.gradle
+++ b/gradle/publish.gradle
@@ -51,6 +51,21 @@ subprojects {
''')
elem.insertBefore(hdr, elem.getFirstChild())
+
+
+ //This black magic checks to see if a dependency has the flag ext.optional=true
+ //set on it, and if so marks the dependency as optional in the maven pom
+ def depMap = project.configurations.compile.dependencies.collectEntries { [it.name, it] }
+ asNode().dependencies.dependency.findAll {
+ def dep = depMap.get(it.artifactId.text())
+ return dep?.hasProperty('optional') && dep.optional
+ }.each {
+ if (it.optional) {
+ it.optional.value = 'true'
+ } else {
+ it.appendNode('optional', 'true')
+ }
+ }
}
project {
[13/14] incubator-geode git commit: Fix up
Posted by kl...@apache.org.
Fix up
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/77147f0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/77147f0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/77147f0e
Branch: refs/heads/feature/GEODE-1050
Commit: 77147f0ec15d2188a8d84e391880a04cbf8b35f7
Parents: def98e3
Author: Kirk Lund <kl...@apache.org>
Authored: Fri Mar 18 14:39:55 2016 -0700
Committer: Kirk Lund <kl...@apache.org>
Committed: Fri Mar 18 14:39:55 2016 -0700
----------------------------------------------------------------------
.../dunit/PartitionedRegionCompactRangeIndexDUnitTest.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/77147f0e/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
index 8be017c..791e320 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
@@ -15,6 +15,7 @@
* limitations under the License.
*/
package com.gemstone.gemfire.cache.query.dunit;
+
import java.io.File;
import java.util.HashMap;
import java.util.Map;
@@ -54,12 +55,12 @@ public class PartitionedRegionCompactRangeIndexDUnitTest extends DistributedTest
return props;
}
- public void setUp() throws Exception {
+ public void postSetUp() throws Exception {
this.disconnectAllFromDS();
}
@Override
- protected void preTearDown() throws Exception {
+ public void preTearDown() throws Exception {
Host host = Host.getHost(0);
VM vm0 = host.getVM(0);
VM vm1 = host.getVM(1);
[05/14] incubator-geode git commit: GEODE-27: Moving findbugs
annotations to geode-core
Posted by kl...@apache.org.
GEODE-27: Moving findbugs annotations to geode-core
These annotations should not be required to run geode. They are only
used by the findbugs tool. Moving them to the geode-core project to
remove the list of dependencies that we add to all projects.
Changing jsr-305.jar to provided because it's only there to avoid
javadoc warnings.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/163046f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/163046f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/163046f2
Branch: refs/heads/feature/GEODE-1050
Commit: 163046f22a3d70699a930f33613161e87dd3e26a
Parents: db17e38
Author: Dan Smith <up...@apache.org>
Authored: Wed Mar 2 14:44:54 2016 -0800
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
geode-assembly/build.gradle | 8 +++++---
geode-core/build.gradle | 3 +++
gradle/java.gradle | 5 -----
3 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/163046f2/geode-assembly/build.gradle
----------------------------------------------------------------------
diff --git a/geode-assembly/build.gradle b/geode-assembly/build.gradle
index 186590e..c393cd8 100755
--- a/geode-assembly/build.gradle
+++ b/geode-assembly/build.gradle
@@ -316,9 +316,11 @@ distributions {
from configurations.bundled
from project(":geode-core").configurations.archives.allArtifacts.files
- from project(":geode-core").webJar
- from project(":geode-core").raJar
- from project(":geode-core").jcaJar
+ //These tasks are included as closures (wrapped in {}) because gradle may evaluate
+ //this CopySpec before it evaluates the geode-core build file.
+ from { project(":geode-core").webJar }
+ from { project(":geode-core").raJar }
+ from { project(":geode-core").jcaJar }
from project(":geode-lucene").configurations.runtime
from project(":geode-lucene").configurations.archives.allArtifacts.files
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/163046f2/geode-core/build.gradle
----------------------------------------------------------------------
diff --git a/geode-core/build.gradle b/geode-core/build.gradle
index 9aa008e..3f4a671 100755
--- a/geode-core/build.gradle
+++ b/geode-core/build.gradle
@@ -31,6 +31,7 @@ dependencies {
// Source Dependencies
// External
provided files("${System.getProperty('java.home')}/../lib/tools.jar")
+ compile 'com.github.stephenc.findbugs:findbugs-annotations:' + project.'stephenc-findbugs.version'
compile 'org.jgroups:jgroups:3.6.7.Final'
compile 'antlr:antlr:' + project.'antlr.version'
compile 'com.fasterxml.jackson.core:jackson-databind:' + project.'jackson.version'
@@ -80,6 +81,8 @@ dependencies {
}
compile 'com.google.guava:guava:' + project.'guava.version'
+ //jsr305 is included only to prevent javadoc warnings about missing annotations in the guava jar
+ provided 'com.google.code.findbugs:jsr305:' + project.'jsr305.version'
compile 'org.apache.logging.log4j:log4j-api:' + project.'log4j.version'
compile 'org.apache.logging.log4j:log4j-core:' + project.'log4j.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/163046f2/gradle/java.gradle
----------------------------------------------------------------------
diff --git a/gradle/java.gradle b/gradle/java.gradle
index 9f0ff25..85e18d8 100644
--- a/gradle/java.gradle
+++ b/gradle/java.gradle
@@ -92,11 +92,6 @@ subprojects {
options.addStringOption('Xdoclint:none', '-quiet')
options.encoding='UTF-8'
}
-
- dependencies {
- compile 'com.github.stephenc.findbugs:findbugs-annotations:' + project.'stephenc-findbugs.version'
- compile 'com.google.code.findbugs:jsr305:' + project.'jsr305.version'
- }
}
[04/14] incubator-geode git commit: GEODE-27: Adding a test to verify
the bundled jars have not changed
Posted by kl...@apache.org.
GEODE-27: Adding a test to verify the bundled jars have not changed
This allows us to make sure we don't accidentally start shipping jars
that we didn't intend to.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/db17e38c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/db17e38c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/db17e38c
Branch: refs/heads/feature/GEODE-1050
Commit: db17e38cc6c66729573d8d91de2c1b61f47f2620
Parents: 02b95b4
Author: Dan Smith <up...@apache.org>
Authored: Tue Mar 15 14:43:07 2016 -0700
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
.../gemstone/gemfire/BundledJarsJUnitTest.java | 129 +++++++++++++++++++
.../src/test/resources/expected_jars.txt | 82 ++++++++++++
2 files changed, 211 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/db17e38c/geode-assembly/src/test/java/com/gemstone/gemfire/BundledJarsJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-assembly/src/test/java/com/gemstone/gemfire/BundledJarsJUnitTest.java b/geode-assembly/src/test/java/com/gemstone/gemfire/BundledJarsJUnitTest.java
new file mode 100644
index 0000000..8ae3b22
--- /dev/null
+++ b/geode-assembly/src/test/java/com/gemstone/gemfire/BundledJarsJUnitTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.jar.JarFile;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.gemstone.gemfire.internal.FileUtil;
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+import com.gemstone.gemfire.util.test.TestUtil;
+
+@Category(UnitTest.class)
+public class BundledJarsJUnitTest {
+
+ private static final String VERSION_PATTERN = "[0-9-_.v]{3,}.*\\.jar$";
+ protected static final String GEMFIRE_HOME = System.getenv("GEMFIRE");
+ private Set<String> expectedJars;
+
+ @Before
+ public void loadExpectedJars() throws IOException {
+ String expectedJarFile = TestUtil.getResourcePath(BundledJarsJUnitTest.class, "/expected_jars.txt");
+
+ expectedJars = Files.lines(Paths.get(expectedJarFile))
+ .collect(Collectors.toSet());
+ }
+
+ @Test
+ public void verifyBundledJarsHaveNotChanged() throws IOException {
+ TreeMap<String, String> sortedJars = getBundledJars();
+ Stream<String> lines = sortedJars.entrySet().stream().map(entry -> removeVersion(entry.getKey()));
+ Set<String> bundledJarNames = new TreeSet<String>(lines.collect(Collectors.toSet()));
+
+ Files.write(Paths.get("bundled_jars.txt"), bundledJarNames);
+
+ TreeSet<String> newJars = new TreeSet<String>(bundledJarNames);
+ newJars.removeAll(expectedJars);
+ TreeSet<String> missingJars = new TreeSet<String>(expectedJars);
+ missingJars.removeAll(bundledJarNames);
+
+ StringBuilder message = new StringBuilder();
+ message.append("The bundled jars have changed. Please make sure you update the licence and notice");
+ message.append("\nas described in https://cwiki.apache.org/confluence/display/GEODE/License+Guide+for+Contributors");
+ message.append("\nWhen fixed, copy geode-assembly/build/test/bundled_jars.txt");
+ message.append("\nto src/test/resources/expected_jars.txt");
+ message.append("\nRemoved Jars\n--------------\n");
+ message.append(String.join("\n", missingJars));
+ message.append("\n\nAdded Jars\n--------------\n");
+ message.append(String.join("\n", newJars));
+ message.append("\n\n");
+
+ assertTrue(message.toString(), expectedJars.equals(bundledJarNames));
+
+ }
+
+ /**
+ * Find all of the jars bundled with the project.
+ * Key is the name of the jar, value is the path.
+ */
+ protected TreeMap<String, String> getBundledJars() {
+ File gemfireHomeDirectory= new File(GEMFIRE_HOME);
+
+ assertTrue("Please set the GEMFIRE environment variable to the product installation directory.",
+ gemfireHomeDirectory.isDirectory());
+
+ List<File> jars = FileUtil.findAll(gemfireHomeDirectory, ".*\\.jar");
+ TreeMap<String, String> sortedJars = new TreeMap<String, String>();
+ jars.stream().forEach(jar -> sortedJars.put(jar.getName(), jar.getPath()));
+
+ List<File> wars = FileUtil.findAll(gemfireHomeDirectory, ".*\\.war");
+ TreeSet<File> sortedWars = new TreeSet<File>(wars);
+ sortedWars.stream().flatMap(BundledJarsJUnitTest::extractJarNames)
+ .forEach(jar -> sortedJars.put(jar.getName(), jar.getPath()));
+
+ sortedJars.keySet().removeIf(s -> s.startsWith("geode"));
+ return sortedJars;
+ }
+
+ private String removeVersion(String name) {
+ return name.replaceAll(VERSION_PATTERN, "");
+ }
+
+ /**
+ * Find of of the jar files embedded within a war
+ */
+ private static Stream<File> extractJarNames(File war) {
+ try (JarFile warContents = new JarFile(war)) {
+ return warContents.stream()
+ //Look for jars in the war
+ .filter(entry -> entry.getName().endsWith(".jar"))
+ //Create a File with a path that includes the war name
+ .map(entry -> new File(war.getName(), entry.getName()))
+ //Materialize the list of files while the war is still open
+ .collect(Collectors.toList()).stream();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/db17e38c/geode-assembly/src/test/resources/expected_jars.txt
----------------------------------------------------------------------
diff --git a/geode-assembly/src/test/resources/expected_jars.txt b/geode-assembly/src/test/resources/expected_jars.txt
new file mode 100644
index 0000000..a566c4d
--- /dev/null
+++ b/geode-assembly/src/test/resources/expected_jars.txt
@@ -0,0 +1,82 @@
+activation
+antlr
+classmate
+commons-beanutils
+commons-collections
+commons-digester
+commons-fileupload
+commons-io
+commons-lang
+commons-logging
+commons-modeler
+fastutil
+findbugs-annotations
+gfsh-dependencies.jar
+guava
+jackson-annotations
+jackson-core
+jackson-databind
+jackson-module-scala
+jansi
+javax.mail-api
+javax.resource-api
+javax.servlet-api
+javax.transaction-api
+jetty-http
+jetty-io
+jetty-security
+jetty-server
+jetty-servlet
+jetty-util
+jetty-webapp
+jetty-xml
+jgroups
+jline
+jna
+json4s-ast
+json4s-core
+json4s-ext
+json4s-jackson
+json4s-native
+log4j-api
+log4j-core
+log4j-jcl
+log4j-jul
+log4j-slf4j-impl
+lucene-analyzers-common
+lucene-core
+lucene-queries
+lucene-queryparser
+mx4j
+mx4j-remote
+mx4j-tools
+netty-all
+paranamer
+ra.jar
+scala-library
+scala-reflect
+slf4j-api
+snappy-java
+spring-aop
+spring-aspects
+spring-beans
+spring-context
+spring-context-support
+spring-core
+spring-data-commons
+spring-data-gemfire
+spring-expression
+spring-hateoas
+spring-ldap-core
+spring-oxm
+spring-security-config
+spring-security-core
+spring-security-ldap
+spring-security-web
+spring-shell
+spring-tx
+spring-web
+spring-webmvc
+swagger-annotations
+swagger-core
+swagger-springmvc
[11/14] incubator-geode git commit: GEODE-1044: Adding license header
to added xml file
Posted by kl...@apache.org.
GEODE-1044: Adding license header to added xml file
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/09cd3302
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/09cd3302
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/09cd3302
Branch: refs/heads/feature/GEODE-1050
Commit: 09cd330258f9876974d2b5d6a2ca7a638656b2bb
Parents: 8310fd9
Author: Jason Huynh <hu...@gmail.com>
Authored: Fri Mar 18 13:55:08 2016 -0700
Committer: Jason Huynh <hu...@gmail.com>
Committed: Fri Mar 18 13:55:34 2016 -0700
----------------------------------------------------------------------
.../query/dunit/PersistentPartitionWithIndex.xml | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/09cd3302/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
----------------------------------------------------------------------
diff --git a/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml b/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
index 8d9fd97..b54df15 100644
--- a/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
+++ b/geode-core/src/test/resources/com/gemstone/gemfire/cache/query/dunit/PersistentPartitionWithIndex.xml
@@ -1,4 +1,20 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<!DOCTYPE cache PUBLIC
"-//GemStone Systems, Inc.//GemFire Declarative Caching 8.0//EN"
"http://www.gemstone.com/dtd/cache8_0.dtd">
[14/14] incubator-geode git commit: Remove this from static method
Posted by kl...@apache.org.
Remove this from static method
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/ae7bf164
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/ae7bf164
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/ae7bf164
Branch: refs/heads/feature/GEODE-1050
Commit: ae7bf16420295887b1048bed1de4954d5a442080
Parents: 77147f0
Author: Kirk Lund <kl...@apache.org>
Authored: Fri Mar 18 14:43:08 2016 -0700
Committer: Kirk Lund <kl...@apache.org>
Committed: Fri Mar 18 14:43:08 2016 -0700
----------------------------------------------------------------------
.../query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ae7bf164/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
index 791e320..3e746e7 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache/query/dunit/PartitionedRegionCompactRangeIndexDUnitTest.java
@@ -56,7 +56,7 @@ public class PartitionedRegionCompactRangeIndexDUnitTest extends DistributedTest
}
public void postSetUp() throws Exception {
- this.disconnectAllFromDS();
+ disconnectAllFromDS();
}
@Override
[08/14] incubator-geode git commit: GEODE-27: Removing a bogus
include of a jetty class
Posted by kl...@apache.org.
GEODE-27: Removing a bogus include of a jetty class
This include was not actually used
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/e2695a5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/e2695a5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/e2695a5b
Branch: refs/heads/feature/GEODE-1050
Commit: e2695a5b5ec0f244a091746108898bb6b4069d69
Parents: 5839e4d
Author: Dan Smith <up...@apache.org>
Authored: Wed Mar 9 16:45:23 2016 -0800
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
.../gemfire/cache/query/internal/CumulativeNonDistinctResults.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e2695a5b/geode-core/src/main/java/com/gemstone/gemfire/cache/query/internal/CumulativeNonDistinctResults.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/cache/query/internal/CumulativeNonDistinctResults.java b/geode-core/src/main/java/com/gemstone/gemfire/cache/query/internal/CumulativeNonDistinctResults.java
index 3735956..095f741 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/cache/query/internal/CumulativeNonDistinctResults.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/cache/query/internal/CumulativeNonDistinctResults.java
@@ -27,8 +27,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import org.eclipse.jetty.webapp.MetaData;
-
import com.gemstone.gemfire.DataSerializer;
import com.gemstone.gemfire.cache.CacheException;
import com.gemstone.gemfire.cache.query.QueryException;
[09/14] incubator-geode git commit: GEODE-27: Enabling transitive
dependencies in the build
Posted by kl...@apache.org.
GEODE-27: Enabling transitive dependencies in the build
Removing the code from the build that disabled transitive dependencies.
Adding excludes and forcing some versions to clean up the list of jars
we are pulling in to match what it was before I enabled transitive
dependencies.
A couple of versions don't match because the versions we were pulling in
before were not consistent across submodules for commons logging and
commons-beanutils.
Some of these excludes, and all of the forced versions look suspect.
They need to be reviewed to see if we really need them.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/dc94a890
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/dc94a890
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/dc94a890
Branch: refs/heads/feature/GEODE-1050
Commit: dc94a890d94381e9fec1b55f3523aa5c2ab379e0
Parents: e2695a5
Author: Dan Smith <up...@apache.org>
Authored: Thu Mar 3 10:35:07 2016 -0800
Committer: Dan Smith <up...@apache.org>
Committed: Fri Mar 18 13:43:55 2016 -0700
----------------------------------------------------------------------
extensions/geode-modules-hibernate/build.gradle | 15 +++-
extensions/geode-modules-tomcat7/build.gradle | 11 +--
extensions/geode-modules/build.gradle | 7 +-
geode-assembly/build.gradle | 14 +++-
geode-core/build.gradle | 72 +++++++++++---------
geode-junit/build.gradle | 4 +-
geode-lucene/build.gradle | 4 +-
geode-pulse/build.gradle | 25 ++++---
geode-rebalancer/build.gradle | 9 ++-
geode-web-api/build.gradle | 56 ++++++++++-----
geode-web/build.gradle | 30 +++++---
gradle/dependency-resolution.gradle | 26 ++++---
gradle/java.gradle | 27 --------
13 files changed, 179 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/extensions/geode-modules-hibernate/build.gradle
----------------------------------------------------------------------
diff --git a/extensions/geode-modules-hibernate/build.gradle b/extensions/geode-modules-hibernate/build.gradle
index 34a76eb..5169b04 100644
--- a/extensions/geode-modules-hibernate/build.gradle
+++ b/extensions/geode-modules-hibernate/build.gradle
@@ -17,11 +17,20 @@
dependencies {
compile project(':extensions/geode-modules')
- compile 'org.hibernate:hibernate-core:' + project.'hibernate.version'
- compile 'org.hibernate:hibernate-annotations:' + project.'hibernate.version'
+ compile ('org.hibernate:hibernate-annotations:' + project.'hibernate.version') {
+ //TODO - do we want these extensions, especially since the show up in testruntime?
+ //If they were needed by our tests, doesn't that mean they will be needed in production?
+ exclude module: 'commons-collections'
+ exclude module: 'dom4j'
+ exclude module: 'hibernate-commons-annotations'
+ exclude module: 'hibernate-jpa-2.0-api'
+ exclude module: 'jta'
+ }
compile 'org.eclipse.persistence:javax.persistence:' + project.'javax.persistence-api.version'
- runtime 'dom4j:dom4j:' + project.'dom4j.version'
+ runtime ('dom4j:dom4j:' + project.'dom4j.version') {
+ exclude module: 'xml-apis'
+ }
testRuntime 'commons-collections:commons-collections:' + project.'commons-collections.version'
testRuntime 'org.hibernate:hibernate-commons-annotations:' + project.'hibernate-commons-annotations.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/extensions/geode-modules-tomcat7/build.gradle
----------------------------------------------------------------------
diff --git a/extensions/geode-modules-tomcat7/build.gradle b/extensions/geode-modules-tomcat7/build.gradle
index c45fa48..07619e8 100644
--- a/extensions/geode-modules-tomcat7/build.gradle
+++ b/extensions/geode-modules-tomcat7/build.gradle
@@ -30,10 +30,13 @@ dependencies {
exclude group: 'org.apache.tomcat'
}
- compile 'org.apache.tomcat:tomcat-api:' + project.'tomcat7.version'
- compile 'org.apache.tomcat:tomcat-catalina:' + project.'tomcat7.version'
- compile 'org.apache.tomcat:tomcat-coyote:' + project.'tomcat7.version'
- compile 'org.apache.tomcat:tomcat-util:' + project.'tomcat7.version'
+ compile ('org.apache.tomcat:tomcat-catalina:' + project.'tomcat7.version') {
+ exclude module: 'tomcat-annotations-api'
+ exclude module: 'tomcat-servlet-api'
+ }
+ compile ('org.apache.tomcat:tomcat-coyote:' + project.'tomcat7.version') {
+ exclude module: 'tomcat-servlet-api'
+ }
compile 'org.apache.tomcat:tomcat-juli:' + project.'tomcat7.version'
testCompile 'org.httpunit:httpunit:' + project.'httpunit.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/extensions/geode-modules/build.gradle
----------------------------------------------------------------------
diff --git a/extensions/geode-modules/build.gradle b/extensions/geode-modules/build.gradle
index 1389585..e007bca 100644
--- a/extensions/geode-modules/build.gradle
+++ b/extensions/geode-modules/build.gradle
@@ -28,8 +28,11 @@ dependencies {
compile project(':geode-core')
compile 'org.slf4j:slf4j-api:' + project.'slf4j-api.version'
- compile 'org.apache.tomcat:catalina:' + project.'tomcat6.version'
- compile 'org.apache.tomcat:catalina-ha:' + project.'tomcat6.version'
+ compile ('org.apache.tomcat:catalina-ha:' + project.'tomcat6.version') {
+ exclude module: 'annotations-api'
+ exclude module: 'coyote'
+ exclude module: 'tribes'
+ }
compile 'org.apache.tomcat:juli:' + project.'tomcat6.version'
testCompile 'org.httpunit:httpunit:' + project.'httpunit.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-assembly/build.gradle
----------------------------------------------------------------------
diff --git a/geode-assembly/build.gradle b/geode-assembly/build.gradle
index 2213614..186590e 100755
--- a/geode-assembly/build.gradle
+++ b/geode-assembly/build.gradle
@@ -49,8 +49,18 @@ configurations {
dependencies {
provided project(':geode-core')
- bundled 'org.springframework.data:spring-data-commons:' + project.'spring-data-commons.version'
- bundled 'org.springframework.data:spring-data-gemfire:' + project.'spring-data-gemfire.version'
+ bundled ('org.springframework.data:spring-data-commons:' + project.'spring-data-commons.version') {
+ transitive=false
+ }
+ bundled ('org.springframework.data:spring-data-gemfire:' + project.'spring-data-gemfire.version') {
+ transitive=false
+ }
+ bundled ('org.springframework:spring-expression:' + project.'springframework.version') {
+ transitive=false
+ }
+ bundled ('org.springframework:spring-tx:' + project.'springframework.version') {
+ transitive=false
+ }
archives project(':geode-common')
archives project(':geode-json')
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-core/build.gradle
----------------------------------------------------------------------
diff --git a/geode-core/build.gradle b/geode-core/build.gradle
index 6ecedef..fe5ab7e 100755
--- a/geode-core/build.gradle
+++ b/geode-core/build.gradle
@@ -33,56 +33,62 @@ dependencies {
provided files("${System.getProperty('java.home')}/../lib/tools.jar")
compile 'org.jgroups:jgroups:3.6.7.Final'
compile 'antlr:antlr:' + project.'antlr.version'
- compile 'com.fasterxml.jackson.core:jackson-annotations:' + project.'jackson.version'
- compile 'com.fasterxml.jackson.core:jackson-core:' + project.'jackson.version'
compile 'com.fasterxml.jackson.core:jackson-databind:' + project.'jackson.version'
- provided 'com.google.guava:guava:' + project.'guava.version'
compile 'commons-io:commons-io:' + project.'commons-io.version'
compile 'commons-lang:commons-lang:' + project.'commons-lang.version'
- compile 'commons-logging:commons-logging:' + project.'commons-logging.version'
- compile 'commons-modeler:commons-modeler:' + project.'commons-modeler.version'
+ compile ('commons-modeler:commons-modeler:' + project.'commons-modeler.version') {
+ exclude module: 'commons-digester'
+ exclude module: 'commons-logging-api'
+ exclude module: 'mx4j-jmx'
+ exclude module: 'xml-apis'
+ }
compile 'io.netty:netty-all:' + project.'netty-all.version'
compile 'it.unimi.dsi:fastutil:' + project.'fastutil.version'
- compile 'javax.activation:activation:' + project.'activation.version'
compile 'javax.mail:javax.mail-api:' + project.'javax.mail-api.version'
compile 'javax.resource:javax.resource-api:' + project.'javax.resource-api.version'
- compile 'javax.servlet:javax.servlet-api:' + project.'javax.servlet-api.version'
- compile 'javax.transaction:javax.transaction-api:' + project.'javax.transaction-api.version'
compile 'mx4j:mx4j:' + project.'mx4j.version'
compile 'mx4j:mx4j-remote:' + project.'mx4j.version'
compile 'mx4j:mx4j-tools:' + project.'mx4j.version'
compile 'net.java.dev.jna:jna:' + project.'jna.version'
- compile 'jline:jline:' + project.'jline.version'
- provided 'org.apache.hadoop:hadoop-common:' + project.'hadoop.version'
- provided 'org.apache.hadoop:hadoop-annotations:' + project.'hadoop.version'
- provided 'org.apache.hadoop:hadoop-hdfs:' + project.'hadoop.version'
- provided 'org.apache.hadoop:hadoop-mapreduce-client-core:' + project.'hadoop.version'
- provided 'org.apache.hbase:hbase:' + project.'hbase.version'
+ provided ('org.apache.hadoop:hadoop-common:' + project.'hadoop.version') {
+ transitive=false
+ }
+ provided ('org.apache.hadoop:hadoop-annotations:' + project.'hadoop.version') {
+ transitive=false
+ }
+ provided ('org.apache.hadoop:hadoop-hdfs:' + project.'hadoop.version') {
+ transitive=false
+ }
+ provided ('org.apache.hadoop:hadoop-mapreduce-client-core:' + project.'hadoop.version') {
+ transitive=false
+ }
+ provided ('org.apache.hbase:hbase:' + project.'hbase.version') {
+ transitive=false
+ }
+
+ compile 'com.google.guava:guava:' + project.'guava.version'
+
compile 'org.apache.logging.log4j:log4j-api:' + project.'log4j.version'
compile 'org.apache.logging.log4j:log4j-core:' + project.'log4j.version'
+ runtime 'org.fusesource.jansi:jansi:' + project.'jansi.version'
runtime 'org.apache.logging.log4j:log4j-slf4j-impl:' + project.'log4j.version'
runtime 'org.apache.logging.log4j:log4j-jcl:' + project.'log4j.version'
runtime 'org.apache.logging.log4j:log4j-jul:' + project.'log4j.version'
- compile 'org.eclipse.jetty:jetty-http:' + project.'jetty.version'
- compile 'org.eclipse.jetty:jetty-io:' + project.'jetty.version'
- compile 'org.eclipse.jetty:jetty-security:' + project.'jetty.version'
- compile 'org.eclipse.jetty:jetty-server:' + project.'jetty.version'
- compile 'org.eclipse.jetty:jetty-servlet:' + project.'jetty.version'
- compile 'org.eclipse.jetty:jetty-util:' + project.'jetty.version'
compile 'org.eclipse.jetty:jetty-webapp:' + project.'jetty.version'
- compile 'org.eclipse.jetty:jetty-xml:' + project.'jetty.version'
- compile 'org.fusesource.jansi:jansi:' + project.'jansi.version'
- compile 'org.slf4j:slf4j-api:' + project.'slf4j-api.version'
- runtime 'org.springframework:spring-aop:' + project.'springframework.version'
- compile 'org.springframework:spring-beans:' + project.'springframework.version' //only used in AbstractCommandsController
- compile 'org.springframework:spring-context:' + project.'springframework.version'
- compile 'org.springframework:spring-context-support:' + project.'springframework.version'
- compile 'org.springframework:spring-core:' + project.'springframework.version'
- runtime 'org.springframework:spring-expression:' + project.'springframework.version'
- compile 'org.springframework:spring-tx:' + project.'springframework.version'
- compile 'org.springframework:spring-web:' + project.'springframework.version'
- compile 'org.springframework:spring-webmvc:' + project.'springframework.version'
- compile 'org.springframework.shell:spring-shell:' + project.'spring-shell.version'
+ runtime ('org.springframework:spring-aop:' + project.'springframework.version') {
+ exclude module: 'aopalliance'
+ }
+ compile ('org.springframework:spring-webmvc:' + project.'springframework.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'spring-aop'
+ }
+ compile ('org.springframework.shell:spring-shell:' + project.'spring-shell.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'asm'
+ exclude module: 'cglib'
+ exclude module: 'guava'
+ exclude module: 'spring-aop'
+ }
compile 'org.xerial.snappy:snappy-java:' + project.'snappy-java.version'
compile project(':geode-common')
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-junit/build.gradle
----------------------------------------------------------------------
diff --git a/geode-junit/build.gradle b/geode-junit/build.gradle
index 7aa6562..3e4eb22 100755
--- a/geode-junit/build.gradle
+++ b/geode-junit/build.gradle
@@ -17,6 +17,8 @@
dependencies {
testCompile 'commons-lang:commons-lang:' + project.'commons-lang.version'
- compile 'junit:junit:' + project.'junit.version'
+ compile ('junit:junit:' + project.'junit.version') {
+ exclude module: 'hamcrest-core'
+ }
compile 'org.hamcrest:hamcrest-all:' + project.'hamcrest-all.version'
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-lucene/build.gradle
----------------------------------------------------------------------
diff --git a/geode-lucene/build.gradle b/geode-lucene/build.gradle
index 55aa756..c1eeb2e 100644
--- a/geode-lucene/build.gradle
+++ b/geode-lucene/build.gradle
@@ -22,7 +22,9 @@ dependencies {
compile 'org.apache.lucene:lucene-analyzers-common:' + project.'lucene.version'
compile 'org.apache.lucene:lucene-core:' + project.'lucene.version'
compile 'org.apache.lucene:lucene-queries:' + project.'lucene.version'
- compile 'org.apache.lucene:lucene-queryparser:' + project.'lucene.version'
+ compile ('org.apache.lucene:lucene-queryparser:' + project.'lucene.version') {
+ exclude module: 'lucene-sandbox'
+ }
testCompile project(':geode-junit')
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-pulse/build.gradle
----------------------------------------------------------------------
diff --git a/geode-pulse/build.gradle b/geode-pulse/build.gradle
index 6dc7b02..045d903 100755
--- a/geode-pulse/build.gradle
+++ b/geode-pulse/build.gradle
@@ -30,23 +30,30 @@ dependencies {
- runtime 'org.springframework:spring-aop:' + project.'springframework.version'
compile 'org.springframework:spring-beans:' + project.'springframework.version'
- compile 'org.springframework:spring-context:' + project.'springframework.version'
- compile 'org.springframework:spring-core:' + project.'springframework.version'
- compile 'org.springframework:spring-web:' + project.'springframework.version'
- runtime 'org.springframework:spring-webmvc:' + project.'springframework.version'
compile 'commons-beanutils:commons-beanutils:' + project.'commons-beanutils.version'
compile 'commons-collections:commons-collections:' + project.'commons-collections.version'
compile 'commons-digester:commons-digester:' + project.'commons-digester.version'
compile 'commons-lang:commons-lang:' + project.'commons-lang.version'
compile 'org.springframework.ldap:spring-ldap-core:' + project.'spring-ldap-core.version'
- compile 'org.springframework.security:spring-security-config:' + project.'spring-security.version'
- compile 'org.springframework.security:spring-security-core:' + project.'spring-security.version'
runtime 'org.springframework:spring-expression:' + project.'springframework.version'
- compile 'org.springframework.security:spring-security-ldap:' + project.'spring-security.version'
- compile 'org.springframework.security:spring-security-web:' + project.'spring-security.version'
+ compile ('org.springframework.security:spring-security-config:' + project.'spring-security.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'spring-expression'
+ }
+ compile ('org.springframework.security:spring-security-ldap:' + project.'spring-security.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'spring-asm'
+ exclude module: 'spring-expression'
+ exclude module: 'spring-jdbc'
+ }
+ compile ('org.springframework.security:spring-security-web:' + project.'spring-security.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'spring-asm'
+ exclude module: 'spring-expression'
+ exclude module: 'spring-jdbc'
+ }
compile 'org.springframework:spring-tx:' + project.'spring-tx.version'
compile 'com.fasterxml.jackson.core:jackson-annotations:' + project.'jackson.version'
compile 'com.fasterxml.jackson.core:jackson-core:' + project.'jackson.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-rebalancer/build.gradle
----------------------------------------------------------------------
diff --git a/geode-rebalancer/build.gradle b/geode-rebalancer/build.gradle
index b9992e9..c21dbf1 100644
--- a/geode-rebalancer/build.gradle
+++ b/geode-rebalancer/build.gradle
@@ -19,11 +19,16 @@ dependencies {
provided project(':geode-common')
provided project(':geode-core')
- compile 'org.quartz-scheduler:quartz:' + project.'quartz.version'
+ compile ('org.quartz-scheduler:quartz:' + project.'quartz.version') {
+ exclude module: 'c3p0'
+ exclude module: 'slf4j-api'
+ }
testCompile project(':geode-junit')
// the following test dependencies are needed for mocking cache instance
testRuntime 'org.apache.hadoop:hadoop-common:' + project.'hadoop.version'
testRuntime 'org.apache.hadoop:hadoop-hdfs:' + project.'hadoop.version'
- runtime 'org.apache.hbase:hbase:' + project.'hbase.version'
+ testRuntime ('org.apache.hbase:hbase:' + project.'hbase.version') {
+ transitive = false
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-web-api/build.gradle
----------------------------------------------------------------------
diff --git a/geode-web-api/build.gradle b/geode-web-api/build.gradle
index dd5f5ca..6a6599e 100755
--- a/geode-web-api/build.gradle
+++ b/geode-web-api/build.gradle
@@ -20,33 +20,55 @@ apply plugin: 'war'
dependencies {
compile 'commons-lang:commons-lang:' + project.'commons-lang.version'
- compile 'commons-fileupload:commons-fileupload:' + project.'commons-fileupload.version'
+ compile ('commons-fileupload:commons-fileupload:' + project.'commons-fileupload.version') {
+ exclude module: 'commons-io'
+ }
compile 'com.fasterxml:classmate:' + project.'classmate.version'
compile 'com.fasterxml.jackson.core:jackson-annotations:' + project.'jackson.version'
compile 'com.fasterxml.jackson.core:jackson-core:' + project.'jackson.version'
compile 'com.fasterxml.jackson.core:jackson-databind:' + project.'jackson.version'
compile 'com.fasterxml.jackson.module:jackson-module-scala_2.10:' + project.'jackson-module-scala_2.10.version'
compile 'com.google.guava:guava:' + project.'guava.version'
- compile 'com.mangofactory:swagger-springmvc:' + project.'swagger-springmvc.version'
- compile 'com.thoughtworks.paranamer:paranamer:' + project.'paranamer.version'
+ compile ('com.mangofactory:swagger-springmvc:' + project.'swagger-springmvc.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'asm'
+ exclude module: 'cglib'
+ exclude module: 'commons-logging'
+ exclude module: 'jackson-jaxrs-json-provider'
+ exclude module: 'jackson-jaxrs-json-provider'
+ exclude module: 'jackson-module-jsonSchema'
+ exclude module: 'joda-convert'
+ exclude module: 'joda-time'
+ exclude module: 'scalap'
+ exclude module: 'slf4j-api'
+ exclude module: 'spring-aop'
+ exclude module: 'spring-beans'
+ exclude module: 'spring-context'
+ exclude module: 'spring-core'
+ exclude module: 'spring-expression'
+ exclude module: 'spring-webmvc'
+ exclude module: 'spring-web'
+ }
compile 'com.wordnik:swagger-annotations:' + project.'swagger.version'
- compile 'com.wordnik:swagger-core_2.10:' + project.'swagger.version'
compile 'org.json4s:json4s-ast_2.10:' + project.'json4s.version'
- compile 'org.json4s:json4s-ext_2.10:' + project.'json4s.version'
- compile 'org.json4s:json4s-core_2.10:' + project.'json4s.version'
- compile 'org.json4s:json4s-jackson_2.10:' + project.'json4s.version'
- compile 'org.json4s:json4s-native_2.10:' + project.'json4s.version'
compile 'org.scala-lang:scala-library:' + project.'scala.version'
compile 'org.scala-lang:scala-reflect:' + project.'scala.version'
- compile 'org.springframework.hateoas:spring-hateoas:' + project.'spring-hateos.version'
- runtime 'org.springframework:spring-aop:' + project.'springframework.version'
- compile 'org.springframework:spring-aspects:' + project.'springframework.version'
- runtime 'org.springframework:spring-beans:' + project.'springframework.version'
- runtime 'org.springframework:spring-context:' + project.'springframework.version'
- runtime 'org.springframework:spring-expression:' + project.'springframework.version'
- compile 'org.springframework:spring-oxm:' + project.'springframework.version'
- runtime 'org.springframework:spring-web:' + project.'springframework.version'
- runtime 'org.springframework:spring-webmvc:' + project.'springframework.version'
+ compile ('org.springframework.hateoas:spring-hateoas:' + project.'spring-hateos.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'commons-logging'
+ exclude module: 'objenesis'
+ exclude module: 'slf4j-api'
+ exclude module: 'spring-core'
+ }
+ compile ('org.springframework:spring-aspects:' + project.'springframework.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'aspectjweaver'
+ }
+ compile ('org.springframework:spring-oxm:' + project.'springframework.version') {
+ exclude module: 'commons-logging'
+ exclude module: 'spring-core'
+ exclude module: 'spring-beans'
+ }
provided 'javax.servlet:javax.servlet-api:' + project.'javax.servlet-api.version'
provided project(':geode-core')
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/geode-web/build.gradle
----------------------------------------------------------------------
diff --git a/geode-web/build.gradle b/geode-web/build.gradle
index 320782d..1173705 100755
--- a/geode-web/build.gradle
+++ b/geode-web/build.gradle
@@ -18,16 +18,26 @@
apply plugin: 'war'
dependencies {
- runtime 'org.springframework:spring-aspects:' + project.'springframework.version'
- runtime 'org.springframework:spring-aop:' + project.'springframework.version'
- runtime 'org.springframework:spring-beans:' + project.'springframework.version'
- runtime 'org.springframework:spring-context:' + project.'springframework.version'
- runtime 'org.springframework:spring-expression:' + project.'springframework.version'
- runtime 'org.springframework:spring-oxm:' + project.'springframework.version'
- runtime 'org.springframework:spring-web:' + project.'springframework.version'
- runtime 'org.springframework:spring-webmvc:' + project.'springframework.version'
-
- runtime 'commons-fileupload:commons-fileupload:' + project.'commons-fileupload.version'
+ runtime ('org.springframework:spring-aspects:' + project.'springframework.version') {
+ exclude module: 'aspectjweaver'
+ exclude module: 'aopalliance'
+ exclude module: 'spring-core'
+ exclude module: 'spring-expression'
+ }
+ runtime ('org.springframework:spring-oxm:' + project.'springframework.version') {
+ exclude module: 'commons-logging'
+ exclude module: 'spring-beans'
+ exclude module: 'spring-core'
+ }
+ runtime ('org.springframework:spring-webmvc:' + project.'springframework.version') {
+ exclude module: 'aopalliance'
+ exclude module: 'aspectjweaver'
+ exclude module: 'spring-core'
+ exclude module: 'spring-expression'
+ }
+ runtime ('commons-fileupload:commons-fileupload:' + project.'commons-fileupload.version') {
+ exclude module: 'commons-io'
+ }
testCompile 'org.springframework:spring-test:' + project.'springframework.version'
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/gradle/dependency-resolution.gradle
----------------------------------------------------------------------
diff --git a/gradle/dependency-resolution.gradle b/gradle/dependency-resolution.gradle
index 1e2b62b..e7f1914 100644
--- a/gradle/dependency-resolution.gradle
+++ b/gradle/dependency-resolution.gradle
@@ -17,17 +17,23 @@
subprojects {
configurations.all {
resolutionStrategy {
- // failOnVersionConflict()
+ //failOnVersionConflict()
-// exclude module: 'aopalliance'
- // exclude 'asm'
- // exclude 'aspectjweaver'
- // exclude 'cglib'
-
-
- // Force certain versions of transitive dependencies
- // force 'asm:asm-all:3.3.1'
- // force 'asm:asm-all:3.3.1'
+ //Force certain versions of transitive dependencies
+ //These are forced to match our old non-transitive configuration
+ //These force statements should be removed and tested with the automatically
+ //resolved version
+ force 'commons-logging:commons-logging:' + project.'commons-logging.version'
+ force 'org.springframework:spring-context-support:' + project.'springframework.version'
+ force 'org.springframework:spring-context-support:' + project.'springframework.version'
+ force 'org.springframework:spring-context:' + project.'springframework.version'
+ force 'org.springframework:spring-web:' + project.'springframework.version'
+ force 'org.springframework:spring-tx:' + project.'springframework.version'
+ force 'org.springframework:spring-webmvc:' + project.'springframework.version'
+ force 'javax.activation:activation:' + project.'activation.version'
+ force 'commons-beanutils:commons-beanutils:' + project.'commons-beanutils.version'
+ force 'com.thoughtworks.paranamer:paranamer:' + project.'paranamer.version'
+ force 'jline:jline:' + project.'jline.version'
}
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dc94a890/gradle/java.gradle
----------------------------------------------------------------------
diff --git a/gradle/java.gradle b/gradle/java.gradle
index ba1122b..9f0ff25 100644
--- a/gradle/java.gradle
+++ b/gradle/java.gradle
@@ -61,33 +61,6 @@ subprojects {
}
}
- // Here we want to disable all transitive dependencies on external artifacts. This
- // allows us to lock down library versions. However, we want project dependencies to
- // be transitive such that the libraries of a dependent project are automatically included.
- configurations.all {
- dependencies.all { dep ->
- if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)) {
- dep.transitive = false
- }
- }
- }
-
- configurations.testCompile {
- dependencies.all { dep ->
- if (dep instanceof ModuleDependency) {
- dep.transitive = true
- }
- }
- }
-
- configurations.testRuntime {
- dependencies.all { dep ->
- if (dep instanceof ModuleDependency) {
- dep.transitive = true
- }
- }
- }
-
// This ensures that javadoc and source jars also have any prefix paths stripped and will
// be created as libs/foo-sources.jar instead of libs/extensions/foo-sources.jar for example.
tasks.all { task ->