You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ud...@apache.org on 2018/03/30 23:51:37 UTC
[geode] branch feature/GEODE-3926_3 updated (922b5d6 -> cb5b44f)
This is an automated email from the ASF dual-hosted git repository.
udo pushed a change to branch feature/GEODE-3926_3
in repository https://gitbox.apache.org/repos/asf/geode.git.
discard 922b5d6 Amended PartitionedRepositoryManagerJUnitTest.java test
discard 83c68ac Added repos.add back into else statement
discard 654d19f Cosmetic
discard 7c42f56 Some cosmetic changes
discard 6ea3f64 GEODE-3926 Initial commit to add LuceneIndexCreationInProgressException
add 38bfac0 docs: typo repair
add 2a6db7c GEODE-4913: gfsh start server cmd is not recognizing local properties… (#1687)
add 1fc7cf6 GEODE-4973: Add docker-compose to pipeline docker image
add 6d8d212 Merge pull request #1702 from nreich/feature/GEODE-4973
add 03f6d27 GEODE-2321: Pulse application works incorrectly in some locales (#1661)
add 7b347af GEODE-3720: Add additional output pattern to failure assertion for ConnectCommandWithSSLTest (#1704)
add 020e5b5 GEODE-4928 DistributedLockService doesn't work as expected while the dlock grantor is initialized
add 755eaaa GEODE-4929: Add ability for tests to overide the default DiskDirs (#1701)
add a4bae5c GEODE-4934: Throw exception if shutdown fails in GfshRule (#1699)
add c42905c GEODE-4811: Add @Disabled "feature flag" annotation for gfsh commands.
add 4bdd31c GEODE-4928 DistributedLockService doesn't work as expected while the dlock grantor is initialized
add b55215d GEODE-3563: use a timeout for newly created sockets in TcpConduit.run() (#1671)
add 956a671 GEODE-4974: Create single connection pool per cache (#1707)
add ffc9c8b GEODE-4976: Fix misspelling and misattribution in test logging. (#1708)
add 1b4d95a GEODE-4917: Remove dead and redundant code from LinkedStructSet.
add 5fe3c76 GEODE-3820: Remove all invocations of Wait.pause(). (#1694)
add 5809567 GEODE-4817: Add support for SSL to the experimental driver. (#1683)
new 6db8ba7 GEODE-3926 Initial commit to add LuceneIndexCreationInProgressException
new 660e937 Some cosmetic changes
new b9d36b1 Cosmetic
new 9c8f726 Added repos.add back into else statement
new cb5b44f Amended PartitionedRepositoryManagerJUnitTest.java test
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (922b5d6)
\
N -- N -- N refs/heads/feature/GEODE-3926_3 (cb5b44f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ci/docker/Dockerfile | 2 +-
.../jdbc/internal/AbstractJdbcCallback.java | 3 +-
.../jdbc/internal/DataSourceManager.java | 5 +-
.../jdbc/internal/JdbcConnectorService.java | 1 +
.../jdbc/internal/JdbcConnectorServiceImpl.java | 7 +
.../geode/connectors/jdbc/internal/SqlHandler.java | 2 +-
.../jdbc/internal/AbstractJdbcCallbackTest.java | 1 -
.../jdbc/internal/DataSourceManagerUnitTest.java | 14 +-
.../jdbc/internal/JdbcConnectorServiceTest.java | 5 +
.../connectors/jdbc/internal/SqlHandlerTest.java | 2 +-
.../cli/CreateConnectionCommandDUnitTest.java | 2 +-
.../org/apache/geode/cache/AttributesFactory.java | 12 +-
.../cache/query/internal/LinkedStructSet.java | 174 +----------
.../InternalClusterConfigurationService.java | 18 +-
.../distributed/internal/tcpserver/TcpServer.java | 5 +-
.../geode/internal/cache/DiskStoreAttributes.java | 3 +-
.../apache/geode/internal/cache/LocalRegion.java | 3 +-
.../DefaultDiskDirs.java} | 31 +-
.../internal/cache/tier/sockets/AcceptorImpl.java | 3 +-
.../internal/cache/xmlcache/CacheXmlGenerator.java | 3 +-
.../geode/internal/lang/SystemPropertyHelper.java | 21 ++
.../apache/geode/internal/net/SocketCreator.java | 7 +-
.../org/apache/geode/internal/tcp/TCPConduit.java | 37 ++-
.../org/apache/geode/management/cli/Disabled.java | 27 +-
.../management/internal/cli/CommandManager.java | 8 +
.../configuration/domain/Configuration.java | 8 -
.../java/org/apache/geode/TestDataSerializer.java | 4 +-
.../cache/client/internal/LocatorSSLJUnitTest.java | 57 ++++
.../internal/DlockAndTxlockRegressionTest.java | 70 ++---
...a => PRAccessorWithOverflowRegressionTest.java} | 29 +-
.../cache/StartServerWithXmlDUnitTest.java | 88 ++++++
.../DefaultDiskDirsIntegrationTest.java} | 34 +--
.../cache/persistence/DefaultDiskDirsTest.java | 50 ++++
...ava => DataSerializerPropagationDUnitTest.java} | 323 ++++++++++-----------
.../geode/internal/net/DummySocketCreator.java | 8 +-
.../internal/net/SSLSocketIntegrationTest.java | 5 +-
.../geode/internal/net/SocketCreatorJUnitTest.java | 17 ++
.../internal/cli/CommandManagerJUnitTest.java | 103 ++++---
.../CreateAsyncEventQueueCommandDUnitTest.java | 3 +-
.../CreateDefinedIndexesCommandDUnitTest.java | 5 +-
.../test/dunit/rules/DistributedDiskDirRule.java | 275 ++++++++++++++++++
.../apache/geode/test/junit/rules/DiskDirRule.java | 188 ++++++++++++
.../cache/client/internal/cacheserver.truststore | Bin 844 -> 1658 bytes
...[XSD].cache.xml => CacheServerWithZeroPort.xml} | 6 +-
.../geode/internal/cache/PutAllCSDUnitTest.java | 4 +-
.../setting_up_a_multisite_system.html.md.erb | 2 +-
.../geode/experimental/driver/DriverFactory.java | 85 +++++-
.../geode/experimental/driver/ProtobufChannel.java | 65 +++--
.../geode/experimental/driver/ProtobufDriver.java | 13 +-
.../geode/experimental/driver/SocketFactory.java | 203 +++++++++++++
.../apache/geode/experimental/driver/SSLTest.java | 187 ++++++++++++
.../geode/experimental/driver/bogusclient.keystore | Bin
.../geode/experimental/driver/bogusserver.keystore | Bin 0 -> 1299 bytes
.../experimental/driver}/cacheserver.keystore | Bin
.../experimental/driver/cacheserver.truststore | Bin 0 -> 2519 bytes
.../geode/experimental/driver}/client.keystore | Bin
.../geode/experimental/driver}/client.truststore | Bin
.../junit/rules/DescribedExternalResource.java | 23 +-
.../geode/test/junit/rules/gfsh/GfshRule.java | 31 +-
.../serializable/SerializableTemporaryFolder.java | 8 +-
.../rules/serializable/SerializableTestName.java | 2 +-
.../tools/pulse/internal/data/PulseConstants.java | 16 +-
.../internal/service/ClusterDetailsService.java | 6 +-
.../service/ClusterMembersRGraphService.java | 13 +-
.../internal/service/ClusterRegionService.java | 7 +-
.../internal/service/ClusterRegionsService.java | 7 +-
.../service/ClusterSelectedRegionService.java | 15 +-
.../internal/service/MemberDetailsService.java | 9 +-
.../internal/service/MemberRegionsService.java | 7 +-
.../geode/tools/pulse/tests/ui/PulseBase.java | 7 +-
.../cli/commands/ConnectCommandWithSSLTest.java | 4 +-
71 files changed, 1711 insertions(+), 672 deletions(-)
copy geode-core/src/main/java/org/apache/geode/internal/cache/{FutureCachedDeserializable.java => persistence/DefaultDiskDirs.java} (54%)
copy geode-junit/src/main/java/org/apache/geode/test/junit/Repeat.java => geode-core/src/main/java/org/apache/geode/management/cli/Disabled.java (61%)
mode change 100755 => 100644
create mode 100644 geode-core/src/test/java/org/apache/geode/cache/client/internal/LocatorSSLJUnitTest.java
rename geode-core/src/test/java/org/apache/geode/internal/cache/{BucketRegionSizeWithOverflowRegressionTest.java => PRAccessorWithOverflowRegressionTest.java} (79%)
create mode 100644 geode-core/src/test/java/org/apache/geode/internal/cache/StartServerWithXmlDUnitTest.java
copy geode-core/src/test/java/org/apache/geode/{distributed/internal/CacheConfigIntegrationTest.java => internal/cache/persistence/DefaultDiskDirsIntegrationTest.java} (56%)
create mode 100644 geode-core/src/test/java/org/apache/geode/internal/cache/persistence/DefaultDiskDirsTest.java
rename geode-core/src/test/java/org/apache/geode/internal/cache/tier/sockets/{DataSerializerPropogationDUnitTest.java => DataSerializerPropagationDUnitTest.java} (80%)
create mode 100644 geode-core/src/test/java/org/apache/geode/test/dunit/rules/DistributedDiskDirRule.java
create mode 100644 geode-core/src/test/java/org/apache/geode/test/junit/rules/DiskDirRule.java
copy geode-core/src/test/resources/org/apache/geode/internal/cache/{wan/GatewayReceiverXmlParsingValidationsJUnitTest.multipleReceiversShouldThrowException[XSD].cache.xml => CacheServerWithZeroPort.xml} (90%)
create mode 100644 geode-experimental-driver/src/main/java/org/apache/geode/experimental/driver/SocketFactory.java
create mode 100644 geode-experimental-driver/src/test/java/org/apache/geode/experimental/driver/SSLTest.java
copy geode-protobuf/src/test/resources/org/apache/geode/internal/protocol/protobuf/v1/acceptance/default.keystore => geode-experimental-driver/src/test/resources/org/apache/geode/experimental/driver/bogusclient.keystore (100%)
create mode 100644 geode-experimental-driver/src/test/resources/org/apache/geode/experimental/driver/bogusserver.keystore
copy {geode-core/src/test/resources/org/apache/geode/cache/client/internal => geode-experimental-driver/src/test/resources/org/apache/geode/experimental/driver}/cacheserver.keystore (100%)
create mode 100644 geode-experimental-driver/src/test/resources/org/apache/geode/experimental/driver/cacheserver.truststore
copy {geode-core/src/test/resources/org/apache/geode/cache/client/internal => geode-experimental-driver/src/test/resources/org/apache/geode/experimental/driver}/client.keystore (100%)
copy {geode-core/src/test/resources/org/apache/geode/cache/client/internal => geode-experimental-driver/src/test/resources/org/apache/geode/experimental/driver}/client.truststore (100%)
--
To stop receiving notification emails like this one, please contact
udo@apache.org.
[geode] 01/05: GEODE-3926 Initial commit to add
LuceneIndexCreationInProgressException
Posted by ud...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
udo pushed a commit to branch feature/GEODE-3926_3
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 6db8ba7c612cd52beade3ac56a35dd7eb3d4b1e4
Author: Udo <uk...@pivotal.io>
AuthorDate: Mon Mar 26 10:51:16 2018 -0700
GEODE-3926 Initial commit to add LuceneIndexCreationInProgressException
---
.../sanctioned-geode-core-serializables.txt | 1 +
.../cache/lucene/internal/InternalLuceneIndex.java | 2 +
...=> LuceneIndexCreationInProgressException.java} | 9 +-
.../internal/LuceneIndexForPartitionedRegion.java | 15 +-
.../cache/lucene/internal/LuceneRawIndex.java | 15 +-
.../lucene/internal/LuceneRawIndexFactory.java | 8 +-
.../cache/lucene/internal/LuceneServiceImpl.java | 6 +-
.../internal/PartitionedRepositoryManager.java | 35 ++++-
.../lucene/internal/RawIndexRepositoryFactory.java | 11 +-
.../internal/RawLuceneRepositoryManager.java | 9 +-
.../sanctioned-geode-lucene-serializables.txt | 1 +
.../lucene/LuceneIndexCreationIntegrationTest.java | 9 +-
.../LuceneIndexRecoveryHAIntegrationTest.java | 6 +-
.../internal/LuceneServiceImplJUnitTest.java | 5 +
.../PartitionedRepositoryManagerJUnitTest.java | 151 ++++++++++++++-------
.../RawLuceneRepositoryManagerJUnitTest.java | 21 +--
16 files changed, 219 insertions(+), 85 deletions(-)
diff --git a/geode-core/src/main/resources/org/apache/geode/internal/sanctioned-geode-core-serializables.txt b/geode-core/src/main/resources/org/apache/geode/internal/sanctioned-geode-core-serializables.txt
index d296f77..c2d1aa4 100644
--- a/geode-core/src/main/resources/org/apache/geode/internal/sanctioned-geode-core-serializables.txt
+++ b/geode-core/src/main/resources/org/apache/geode/internal/sanctioned-geode-core-serializables.txt
@@ -803,3 +803,4 @@ org/apache/geode/security/AuthenticationFailedException,true,-820286647227908887
org/apache/geode/security/AuthenticationRequiredException,true,4675976651103154919
org/apache/geode/security/GemFireSecurityException,true,3814254578203076926,cause:java/lang/Throwable
org/apache/geode/security/NotAuthorizedException,true,419215768216387745,principal:java/security/Principal
+org/apache/geode/cache/lucene/internal/LuceneIndexCreationInProgressException,false
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/InternalLuceneIndex.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/InternalLuceneIndex.java
index 74e4ac8..1e95a46 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/InternalLuceneIndex.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/InternalLuceneIndex.java
@@ -39,4 +39,6 @@ public interface InternalLuceneIndex extends LuceneIndex {
void initialize();
+ boolean isIndexAvailable(int id);
+
}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexCreationInProgressException.java
old mode 100755
new mode 100644
similarity index 74%
copy from geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java
copy to geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexCreationInProgressException.java
index 4a92049..9c58ba5
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexCreationInProgressException.java
@@ -14,11 +14,10 @@
*/
package org.apache.geode.cache.lucene.internal;
-import org.apache.geode.internal.cache.InternalCache;
+import org.apache.geode.GemFireException;
-public class LuceneRawIndexFactory extends LuceneIndexImplFactory {
- @Override
- public LuceneIndexImpl create(String indexName, String regionPath, InternalCache cache) {
- return new LuceneRawIndex(indexName, regionPath, cache);
+public class LuceneIndexCreationInProgressException extends GemFireException {
+ public LuceneIndexCreationInProgressException(String message) {
+ super(message);
}
}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
index 577bdef..032f1b7 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
@@ -16,6 +16,7 @@
package org.apache.geode.cache.lucene.internal;
import java.util.Set;
+import java.util.concurrent.ExecutorService;
import org.apache.geode.CancelException;
import org.apache.geode.cache.AttributesFactory;
@@ -49,8 +50,11 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
public static final String FILES_REGION_SUFFIX = ".files";
+ private ExecutorService waitingThreadPoolFromDM;
+
public LuceneIndexForPartitionedRegion(String indexName, String regionPath, InternalCache cache) {
super(indexName, regionPath, cache);
+ this.waitingThreadPoolFromDM = cache.getDistributionManager().getWaitingThreadPool();
final String statsName = indexName + "-" + regionPath;
this.fileSystemStats = new FileSystemStats(cache.getDistributedSystem(), statsName);
@@ -62,7 +66,7 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
mapper = new HeterogeneousLuceneSerializer();
}
PartitionedRepositoryManager partitionedRepositoryManager =
- new PartitionedRepositoryManager(this, mapper);
+ new PartitionedRepositoryManager(this, mapper, this.waitingThreadPoolFromDM);
return partitionedRepositoryManager;
}
@@ -202,6 +206,15 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
}
}
+ @Override
+ public boolean isIndexAvailable(int id) {
+ PartitionedRegion fileAndChunkRegion = getFileAndChunkRegion();
+ if (fileAndChunkRegion != null) {
+ return fileAndChunkRegion.get(IndexRepositoryFactory.APACHE_GEODE_INDEX_COMPLETE, id) != null;
+ }
+ return false;
+ }
+
private void destroyOnRemoteMembers() {
PartitionedRegion pr = (PartitionedRegion) getDataRegion();
DistributionManager dm = pr.getDistributionManager();
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java
index d4168bd..6bf7a23 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java
@@ -22,8 +22,12 @@ import org.apache.geode.internal.cache.PartitionedRegion;
public class LuceneRawIndex extends LuceneIndexImpl {
- protected LuceneRawIndex(String indexName, String regionPath, InternalCache cache) {
+ private String luceneFolderPath;
+
+ protected LuceneRawIndex(String indexName, String regionPath, InternalCache cache,
+ String luceneFolderPath) {
super(indexName, regionPath, cache);
+ this.luceneFolderPath = luceneFolderPath;
}
@Override
@@ -32,8 +36,8 @@ public class LuceneRawIndex extends LuceneIndexImpl {
if (mapper == null) {
mapper = new HeterogeneousLuceneSerializer();
}
- RawLuceneRepositoryManager rawLuceneRepositoryManager =
- new RawLuceneRepositoryManager(this, mapper);
+ RawLuceneRepositoryManager rawLuceneRepositoryManager = new RawLuceneRepositoryManager(this,
+ mapper, cache.getDistributionManager().getWaitingThreadPool(), luceneFolderPath);
return rawLuceneRepositoryManager;
}
@@ -50,4 +54,9 @@ public class LuceneRawIndex extends LuceneIndexImpl {
@Override
public void destroy(boolean initiator) {}
+
+ @Override
+ public boolean isIndexAvailable(int id) {
+ return true;
+ }
}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java
index 4a92049..1f91af8 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java
@@ -17,8 +17,14 @@ package org.apache.geode.cache.lucene.internal;
import org.apache.geode.internal.cache.InternalCache;
public class LuceneRawIndexFactory extends LuceneIndexImplFactory {
+ private String luceneFolderPath;
+
+ public LuceneRawIndexFactory(String luceneFolderPath) {
+ this.luceneFolderPath = luceneFolderPath;
+ }
+
@Override
public LuceneIndexImpl create(String indexName, String regionPath, InternalCache cache) {
- return new LuceneRawIndex(indexName, regionPath, cache);
+ return new LuceneRawIndex(indexName, regionPath, cache, luceneFolderPath);
}
}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
index 01bc5c6..3e08abd 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
@@ -166,8 +166,7 @@ public class LuceneServiceImpl implements InternalLuceneService {
if (!regionPath.startsWith("/")) {
regionPath = "/" + regionPath;
}
- String name = indexName + "#" + regionPath.replace('/', '_');
- return name;
+ return indexName + "#" + regionPath.replace('/', '_');
}
public static String getUniqueIndexRegionName(String indexName, String regionPath,
@@ -182,7 +181,7 @@ public class LuceneServiceImpl implements InternalLuceneService {
}
Analyzer analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(), fieldAnalyzers);
Set<String> fieldsSet = fieldAnalyzers.keySet();
- String[] fields = fieldsSet.toArray(new String[fieldsSet.size()]);
+ String[] fields = fieldsSet.toArray(new String[0]);
createIndex(indexName, regionPath, analyzer, fieldAnalyzers, serializer, allowOnExistingRegion,
fields);
@@ -209,6 +208,7 @@ public class LuceneServiceImpl implements InternalLuceneService {
return;
}
+
if (!allowOnExistingRegion) {
definedIndexMap.remove(LuceneServiceImpl.getUniqueIndexName(indexName, regionPath));
throw new IllegalStateException("The lucene index must be created before region");
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
index f60f83b..92b4c08 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
@@ -20,6 +20,9 @@ import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.logging.log4j.Logger;
import org.apache.geode.InternalGemFireError;
import org.apache.geode.cache.Region;
@@ -32,8 +35,10 @@ import org.apache.geode.internal.cache.BucketNotFoundException;
import org.apache.geode.internal.cache.BucketRegion;
import org.apache.geode.internal.cache.PartitionedRegion;
import org.apache.geode.internal.cache.execute.InternalRegionFunctionContext;
+import org.apache.geode.internal.logging.LogService;
public class PartitionedRepositoryManager implements RepositoryManager {
+ private final Logger logger = LogService.getLogger();
public static IndexRepositoryFactory indexRepositoryFactory = new IndexRepositoryFactory();
/**
* map of the parent bucket region to the index repository
@@ -47,17 +52,23 @@ public class PartitionedRepositoryManager implements RepositoryManager {
protected final ConcurrentHashMap<Integer, IndexRepository> indexRepositories =
new ConcurrentHashMap<Integer, IndexRepository>();
- /** The user region for this index */
+ /**
+ * The user region for this index
+ */
protected PartitionedRegion userRegion = null;
protected final LuceneSerializer serializer;
protected final InternalLuceneIndex index;
protected volatile boolean closed;
private final CountDownLatch isDataRegionReady = new CountDownLatch(1);
- public PartitionedRepositoryManager(InternalLuceneIndex index, LuceneSerializer serializer) {
+ private final ExecutorService waitingThreadPoolFromDM;
+
+ public PartitionedRepositoryManager(InternalLuceneIndex index, LuceneSerializer serializer,
+ ExecutorService waitingThreadPool) {
this.index = index;
this.serializer = serializer;
this.closed = false;
+ this.waitingThreadPoolFromDM = waitingThreadPool;
}
public void setUserRegionForRepositoryManager(PartitionedRegion userRegion) {
@@ -70,13 +81,28 @@ public class PartitionedRepositoryManager implements RepositoryManager {
Region<Object, Object> region = ctx.getDataSet();
Set<Integer> buckets = ((InternalRegionFunctionContext) ctx).getLocalBucketSet(region);
ArrayList<IndexRepository> repos = new ArrayList<IndexRepository>(buckets.size());
+
for (Integer bucketId : buckets) {
BucketRegion userBucket = userRegion.getDataStore().getLocalBucketById(bucketId);
if (userBucket == null) {
throw new BucketNotFoundException(
"User bucket was not found for region " + region + "bucket id " + bucketId);
} else {
- repos.add(getRepository(userBucket.getId()));
+ if (index.isIndexAvailable(userBucket.getId())) {
+ repos.add(getRepository(userBucket.getId()));
+ } else {
+ waitingThreadPoolFromDM.execute(() -> {
+ try {
+ IndexRepository repository = getRepository(userBucket.getId());
+ repos.add(repository);
+ } catch (BucketNotFoundException e) {
+ logger.debug(
+ "Lucene Index creation in progress. Catching BucketNotFoundException");
+ }
+ });
+ throw new LuceneIndexCreationInProgressException(
+ "Lucene Index creation in progress for bucket: " + userBucket.getId());
+ }
}
}
@@ -155,7 +181,8 @@ public class PartitionedRepositoryManager implements RepositoryManager {
try {
computeRepository(bucketId);
} catch (LuceneIndexDestroyedException e) {
- /* expected exception */}
+ /* expected exception */
+ }
}
}
}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java
index 984d3eb..ea598bb 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java
@@ -30,6 +30,9 @@ import org.apache.geode.internal.cache.BucketRegion;
import org.apache.geode.internal.cache.PartitionedRegion;
public class RawIndexRepositoryFactory extends IndexRepositoryFactory {
+
+ private String luceneFolderPath = "";
+
public RawIndexRepositoryFactory() {}
@Override
@@ -46,7 +49,9 @@ public class RawIndexRepositoryFactory extends IndexRepositoryFactory {
if (indexForRaw.withPersistence()) {
String bucketLocation = LuceneServiceImpl.getUniqueIndexName(index.getName(),
index.getRegionPath() + "_" + bucketId);
- File location = new File(index.getName(), bucketLocation);
+ String filePath = luceneFolderPath.isEmpty() ? index.getName()
+ : luceneFolderPath + File.separator + index.getName();
+ File location = new File(filePath, bucketLocation);
if (!location.exists()) {
location.mkdirs();
}
@@ -60,4 +65,8 @@ public class RawIndexRepositoryFactory extends IndexRepositoryFactory {
return new IndexRepositoryImpl(null, writer, serializer, indexForRaw.getIndexStats(),
dataBucket, null, "", indexForRaw);
}
+
+ public void setLuceneFolderPath(String luceneFolderPath) {
+ this.luceneFolderPath = luceneFolderPath;
+ }
}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java
index 0b38c45..c297159 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java
@@ -15,6 +15,7 @@
package org.apache.geode.cache.lucene.internal;
import java.io.IOException;
+import java.util.concurrent.ExecutorService;
import org.apache.geode.cache.lucene.LuceneSerializer;
import org.apache.geode.cache.lucene.internal.repository.IndexRepository;
@@ -22,10 +23,12 @@ import org.apache.geode.internal.cache.BucketNotFoundException;
import org.apache.geode.internal.cache.PartitionedRegion;
public class RawLuceneRepositoryManager extends PartitionedRepositoryManager {
- public static IndexRepositoryFactory indexRepositoryFactory = new RawIndexRepositoryFactory();
+ public static RawIndexRepositoryFactory indexRepositoryFactory = new RawIndexRepositoryFactory();
- public RawLuceneRepositoryManager(LuceneIndexImpl index, LuceneSerializer serializer) {
- super(index, serializer);
+ public RawLuceneRepositoryManager(LuceneIndexImpl index, LuceneSerializer serializer,
+ ExecutorService waitingThreadPool, String luceneFolderPath) {
+ super(index, serializer, waitingThreadPool);
+ indexRepositoryFactory.setLuceneFolderPath(luceneFolderPath);
}
@Override
diff --git a/geode-lucene/src/main/resources/org/apache/geode/internal/sanctioned-geode-lucene-serializables.txt b/geode-lucene/src/main/resources/org/apache/geode/internal/sanctioned-geode-lucene-serializables.txt
index a13c06b..3a9117a 100755
--- a/geode-lucene/src/main/resources/org/apache/geode/internal/sanctioned-geode-lucene-serializables.txt
+++ b/geode-lucene/src/main/resources/org/apache/geode/internal/sanctioned-geode-lucene-serializables.txt
@@ -17,3 +17,4 @@ org/apache/geode/cache/lucene/internal/directory/DumpDirectoryFiles,true,1
org/apache/geode/cache/lucene/internal/distributed/LuceneQueryFunction,true,1
org/apache/geode/cache/lucene/internal/distributed/WaitUntilFlushedFunction,true,1
org/apache/geode/cache/lucene/internal/results/LuceneGetPageFunction,true,1
+org/apache/geode/cache/lucene/internal/LuceneIndexCreationInProgressException,false
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneIndexCreationIntegrationTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneIndexCreationIntegrationTest.java
index 55c2222..80310ab 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneIndexCreationIntegrationTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneIndexCreationIntegrationTest.java
@@ -69,6 +69,7 @@ import org.apache.geode.internal.cache.LocalRegion;
import org.apache.geode.internal.cache.PartitionedRegion;
import org.apache.geode.test.junit.categories.IntegrationTest;
import org.apache.geode.test.junit.categories.LuceneTest;
+import org.apache.geode.test.junit.rules.serializable.SerializableTemporaryFolder;
/**
* Tests of creating lucene indexes on regions. All tests of index creation use cases should be in
@@ -85,6 +86,9 @@ public class LuceneIndexCreationIntegrationTest extends LuceneIntegrationTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
+ @Rule
+ public SerializableTemporaryFolder temporaryFolder = new SerializableTemporaryFolder();
+
@Test
public void shouldCreateIndexWriterWithAnalyzersWhenSettingPerFieldAnalyzers()
@@ -162,14 +166,15 @@ public class LuceneIndexCreationIntegrationTest extends LuceneIntegrationTest {
@Test
public void shouldCreateRawIndexIfSpecifiedItsFactory()
- throws BucketNotFoundException, InterruptedException {
+ throws BucketNotFoundException, InterruptedException, IOException {
Map<String, Analyzer> analyzers = new HashMap<>();
final RecordingAnalyzer field1Analyzer = new RecordingAnalyzer();
final RecordingAnalyzer field2Analyzer = new RecordingAnalyzer();
analyzers.put("field1", field1Analyzer);
analyzers.put("field2", field2Analyzer);
- LuceneServiceImpl.luceneIndexFactory = new LuceneRawIndexFactory();
+ LuceneServiceImpl.luceneIndexFactory =
+ new LuceneRawIndexFactory(temporaryFolder.newFolder("lucene").getPath());
try {
luceneService.createIndexFactory().setFields(analyzers).create(INDEX_NAME, REGION_NAME);
Region region = createRegion();
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexRecoveryHAIntegrationTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexRecoveryHAIntegrationTest.java
index 0f67cb6..599d0a1 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexRecoveryHAIntegrationTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexRecoveryHAIntegrationTest.java
@@ -19,6 +19,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.Analyzer;
@@ -91,7 +92,8 @@ public class LuceneIndexRecoveryHAIntegrationTest {
userRegion.put("rebalance", "test");
service.waitUntilFlushed("index1", "userRegion", 30000, TimeUnit.MILLISECONDS);
- RepositoryManager manager = new PartitionedRepositoryManager((LuceneIndexImpl) index, mapper);
+ RepositoryManager manager =
+ new PartitionedRepositoryManager(index, mapper, Executors.newSingleThreadExecutor());
IndexRepository repo = manager.getRepository(userRegion, 0, null);
assertNotNull(repo);
@@ -106,7 +108,7 @@ public class LuceneIndexRecoveryHAIntegrationTest {
userRegion = (PartitionedRegion) regionfactory.create("userRegion");
userRegion.put("rebalance", "test");
- manager = new PartitionedRepositoryManager((LuceneIndexImpl) index, mapper);
+ manager = new PartitionedRepositoryManager(index, mapper, Executors.newSingleThreadExecutor());
IndexRepository newRepo = manager.getRepository(userRegion, 0, null);
Assert.assertNotEquals(newRepo, repo);
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
index 07c2d6c..8b09ff5 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.Analyzer;
@@ -52,6 +53,7 @@ import org.apache.geode.cache.asyncqueue.internal.AsyncEventQueueFactoryImpl;
import org.apache.geode.cache.lucene.LuceneIndexFactory;
import org.apache.geode.cache.lucene.LuceneSerializer;
import org.apache.geode.distributed.DistributedSystem;
+import org.apache.geode.distributed.internal.DistributionManager;
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.cache.PartitionedRegion;
import org.apache.geode.internal.cache.PartitionedRegionDataStore;
@@ -123,6 +125,9 @@ public class LuceneServiceImplJUnitTest {
when(((StatisticsFactory) ds).createAtomicStatistics(any(), anyString()))
.thenReturn(luceneIndexStats);
when(cache.getRegion(anyString())).thenReturn(region);
+ when(cache.getDistributionManager()).thenReturn(mock(DistributionManager.class));
+ when(cache.getDistributionManager().getWaitingThreadPool())
+ .thenReturn(Executors.newSingleThreadExecutor());
RegionAttributes ratts = mock(RegionAttributes.class);
when(region.getAttributes()).thenReturn(ratts);
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java
index b2c30b1..38319b6 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java
@@ -29,20 +29,24 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter;
+import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
@@ -64,13 +68,13 @@ import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.cache.LocalRegion;
import org.apache.geode.internal.cache.PartitionRegionConfig;
import org.apache.geode.internal.cache.PartitionedRegion;
-import org.apache.geode.internal.cache.PartitionedRegion.RetryTimeKeeper;
import org.apache.geode.internal.cache.PartitionedRegionDataStore;
import org.apache.geode.internal.cache.PartitionedRegionHelper;
import org.apache.geode.internal.cache.execute.InternalRegionFunctionContext;
import org.apache.geode.test.fake.Fakes;
import org.apache.geode.test.junit.categories.LuceneTest;
import org.apache.geode.test.junit.categories.UnitTest;
+import org.apache.geode.test.junit.rules.serializable.SerializableTemporaryFolder;
@Category({UnitTest.class, LuceneTest.class})
@RunWith(PowerMockRunner.class)
@@ -83,20 +87,24 @@ public class PartitionedRepositoryManagerJUnitTest {
protected LuceneSerializer serializer;
protected PartitionedRegionDataStore userDataStore;
protected PartitionedRegionDataStore fileDataStore;
- protected PartitionedRegionHelper prHelper;
protected PartitionRegionConfig prConfig;
protected LocalRegion prRoot;
- protected Map<Integer, BucketRegion> fileAndChunkBuckets = new HashMap<Integer, BucketRegion>();
- protected Map<Integer, BucketRegion> dataBuckets = new HashMap<Integer, BucketRegion>();
+ protected Map<Integer, BucketRegion> fileAndChunkBuckets = new HashMap<>();
+ protected Map<Integer, BucketRegion> dataBuckets = new HashMap<>();
protected LuceneIndexStats indexStats;
protected FileSystemStats fileSystemStats;
protected LuceneIndexImpl indexForPR;
protected PartitionedRepositoryManager repoManager;
protected GemFireCacheImpl cache;
+ private final Map<Integer, Boolean> isIndexAvailableMap = new HashMap<>();
+
+ @Rule
+ public SerializableTemporaryFolder temporaryFolder = new SerializableTemporaryFolder();
+
@Before
- public void setUp() {
+ public void setUp() throws IOException {
cache = Fakes.cache();
userRegion = Mockito.mock(PartitionedRegion.class);
userDataStore = Mockito.mock(PartitionedRegionDataStore.class);
@@ -117,7 +125,10 @@ public class PartitionedRepositoryManagerJUnitTest {
DLockService.removeLockServiceForTests(PartitionedRegionHelper.PARTITION_LOCK_SERVICE_NAME);
}
- protected void createIndexAndRepoManager() {
+ protected void createIndexAndRepoManager() throws IOException {
+ String luceneFolderPath = temporaryFolder.newFolder("lucene").getPath();
+ LuceneServiceImpl.luceneIndexFactory = new LuceneRawIndexFactory(luceneFolderPath);
+
fileAndChunkRegion = Mockito.mock(PartitionedRegion.class);
fileDataStore = Mockito.mock(PartitionedRegionDataStore.class);
when(fileAndChunkRegion.getDataStore()).thenReturn(fileDataStore);
@@ -142,13 +153,14 @@ public class PartitionedRepositoryManagerJUnitTest {
when(prRoot.get("rid")).thenReturn(prConfig);
PowerMockito.mockStatic(PartitionedRegionHelper.class);
PowerMockito.when(PartitionedRegionHelper.getPRRoot(cache)).thenReturn(prRoot);
- repoManager = new PartitionedRepositoryManager(indexForPR, serializer);
+ repoManager = new PartitionedRepositoryManager(indexForPR, serializer,
+ Executors.newSingleThreadExecutor());
repoManager.setUserRegionForRepositoryManager(userRegion);
repoManager.allowRepositoryComputation();
}
@Test
- public void getByKey() throws BucketNotFoundException, IOException {
+ public void getByKey() throws BucketNotFoundException {
setUpMockBucket(0);
setUpMockBucket(1);
@@ -165,23 +177,22 @@ public class PartitionedRepositoryManagerJUnitTest {
assertEquals(repo0, repo113);
assertNotEquals(repo0, repo1);
- checkRepository(repo0, 0);
- checkRepository(repo1, 1);
+ checkRepositoryContainsBucket(repo0, 0);
+ checkRepositoryContainsBucket(repo1, 1);
}
/**
* Test what happens when a bucket is destroyed.
*/
@Test
- public void destroyBucketShouldCreateNewIndexRepository()
- throws BucketNotFoundException, IOException {
+ public void destroyBucketShouldCreateNewIndexRepository() throws BucketNotFoundException {
setUpMockBucket(0);
IndexRepositoryImpl repo0 =
(IndexRepositoryImpl) repoManager.getRepository(userRegion, 0, null);
assertNotNull(repo0);
- checkRepository(repo0, 0);
+ checkRepositoryContainsBucket(repo0, 0);
BucketRegion fileBucket0 = fileAndChunkBuckets.get(0);
BucketRegion dataBucket0 = dataBuckets.get(0);
@@ -194,7 +205,7 @@ public class PartitionedRepositoryManagerJUnitTest {
IndexRepositoryImpl newRepo0 =
(IndexRepositoryImpl) repoManager.getRepository(userRegion, 0, null);
assertNotEquals(repo0, newRepo0);
- checkRepository(newRepo0, 0);
+ checkRepositoryContainsBucket(newRepo0, 0);
assertTrue(repo0.isClosed());
assertFalse(repo0.getWriter().isOpen());
}
@@ -213,45 +224,19 @@ public class PartitionedRepositoryManagerJUnitTest {
when(fileDataStore.getLocalBucketById(eq(0))).thenReturn(null);
- when(fileAndChunkRegion.getOrCreateNodeForBucketWrite(eq(0), (RetryTimeKeeper) any()))
- .then(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- when(fileDataStore.getLocalBucketById(eq(0))).thenReturn(fileAndChunkBuckets.get(0));
- return null;
- }
+ when(fileAndChunkRegion.getOrCreateNodeForBucketWrite(eq(0), any()))
+ .then((Answer) invocation -> {
+ when(fileDataStore.getLocalBucketById(eq(0))).thenReturn(fileAndChunkBuckets.get(0));
+ return null;
});
assertNotNull(repoManager.getRepository(userRegion, 0, null));
}
- @Test
- public void getByRegion() throws BucketNotFoundException {
- setUpMockBucket(0);
- setUpMockBucket(1);
-
- Set<Integer> buckets = new LinkedHashSet<Integer>(Arrays.asList(0, 1));
- InternalRegionFunctionContext ctx = Mockito.mock(InternalRegionFunctionContext.class);
- when(ctx.getLocalBucketSet((any()))).thenReturn(buckets);
- Collection<IndexRepository> repos = repoManager.getRepositories(ctx);
- assertEquals(2, repos.size());
-
- Iterator<IndexRepository> itr = repos.iterator();
- IndexRepositoryImpl repo0 = (IndexRepositoryImpl) itr.next();
- IndexRepositoryImpl repo1 = (IndexRepositoryImpl) itr.next();
-
- assertNotNull(repo0);
- assertNotNull(repo1);
- assertNotEquals(repo0, repo1);
-
- checkRepository(repo0, 0);
- checkRepository(repo1, 1);
- }
-
/**
* Test that we get the expected exception when a user bucket is missing
*/
- @Test(expected = BucketNotFoundException.class)
+ @Test(expected = LuceneIndexCreationInProgressException.class)
public void getMissingBucketByRegion() throws BucketNotFoundException {
setUpMockBucket(0);
@@ -262,15 +247,21 @@ public class PartitionedRepositoryManagerJUnitTest {
repoManager.getRepositories(ctx);
}
- protected void checkRepository(IndexRepositoryImpl repo0, int bucketId) {
+ protected void checkRepositoryContainsBucket(IndexRepositoryImpl repo0, int... bucketIds) {
IndexWriter writer0 = repo0.getWriter();
RegionDirectory dir0 = (RegionDirectory) writer0.getDirectory();
- assertEquals(new BucketTargetingMap(fileAndChunkBuckets.get(bucketId), bucketId),
- dir0.getFileSystem().getFileAndChunkRegion());
+ boolean result = false;
+ for (int bucketId : bucketIds) {
+ BucketTargetingMap bucketTargetingMap =
+ new BucketTargetingMap(fileAndChunkBuckets.get(bucketId), bucketId);
+ result |= bucketTargetingMap.equals(dir0.getFileSystem().getFileAndChunkRegion());
+ }
+
+ assertTrue(result);
assertEquals(serializer, repo0.getSerializer());
}
- protected BucketRegion setUpMockBucket(int id) throws BucketNotFoundException {
+ protected void setUpMockBucket(int id) {
BucketRegion mockBucket = Mockito.mock(BucketRegion.class);
BucketRegion fileAndChunkBucket = Mockito.mock(BucketRegion.class);
// Allowing the fileAndChunkBucket to behave like a map so that the IndexWriter operations don't
@@ -290,6 +281,64 @@ public class PartitionedRepositoryManagerJUnitTest {
BucketAdvisor mockBucketAdvisor = Mockito.mock(BucketAdvisor.class);
when(fileAndChunkBucket.getBucketAdvisor()).thenReturn(mockBucketAdvisor);
when(mockBucketAdvisor.isPrimary()).thenReturn(true);
- return mockBucket;
}
+
+ @Test
+ public void queryByRegionWaitingForRepoToBeCreated() {
+ setUpMockBucket(0);
+ setUpMockBucket(1);
+
+ setupIsIndexAvailable();
+
+ Set<Integer> buckets = new LinkedHashSet<>(Arrays.asList(0, 1));
+ InternalRegionFunctionContext ctx = Mockito.mock(InternalRegionFunctionContext.class);
+ when(ctx.getLocalBucketSet((any()))).thenReturn(buckets);
+ final Collection<IndexRepository> repositories = new HashSet<>();
+
+ Awaitility.await().pollDelay(1, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS)
+ .atMost(500, TimeUnit.SECONDS).until(() -> {
+ try {
+ repositories.addAll(repoManager.getRepositories(ctx));
+ } catch (BucketNotFoundException | LuceneIndexCreationInProgressException e) {
+ }
+ return repositories.size() == 2;
+ });
+
+ Iterator<IndexRepository> itr = repositories.iterator();
+ IndexRepositoryImpl repo0 = (IndexRepositoryImpl) itr.next();
+ IndexRepositoryImpl repo1 = (IndexRepositoryImpl) itr.next();
+
+ assertNotNull(repo0);
+ assertNotNull(repo1);
+ assertNotEquals(repo0, repo1);
+
+ checkRepositoryContainsBucket(repo0, 0, 1);
+ checkRepositoryContainsBucket(repo1, 0, 1);
+ }
+
+ private void setupIsIndexAvailable() {
+ when(indexForPR.isIndexAvailable(1)).then((Answer) invocation -> {
+ boolean result;
+ Boolean isAvailable = isIndexAvailableMap.get(1);
+ if (isAvailable == null || !isAvailable) {
+ isIndexAvailableMap.put(1, true);
+ result = false;
+ } else {
+ result = true;
+ }
+ return result;
+ });
+ when(indexForPR.isIndexAvailable(0)).then((Answer) invocation -> {
+ boolean result;
+ Boolean isAvailable = isIndexAvailableMap.get(0);
+ if (isAvailable == null || !isAvailable) {
+ isIndexAvailableMap.put(0, true);
+ result = false;
+ } else {
+ result = true;
+ }
+ return result;
+ });
+ }
+
}
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManagerJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManagerJUnitTest.java
index a000d2f..ffe24ce 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManagerJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManagerJUnitTest.java
@@ -19,6 +19,9 @@ import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.when;
+import java.io.IOException;
+import java.util.concurrent.Executors;
+
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
@@ -35,12 +38,11 @@ import org.apache.geode.internal.cache.BucketRegion;
import org.apache.geode.internal.cache.PartitionedRegion;
import org.apache.geode.internal.cache.PartitionedRegionDataStore;
import org.apache.geode.test.fake.Fakes;
-import org.apache.geode.test.junit.categories.LuceneTest;
public class RawLuceneRepositoryManagerJUnitTest extends PartitionedRepositoryManagerJUnitTest {
@Before
- public void setUp() {
+ public void setUp() throws IOException {
cache = Fakes.cache();
userRegion = Mockito.mock(PartitionedRegion.class);
@@ -53,11 +55,12 @@ public class RawLuceneRepositoryManagerJUnitTest extends PartitionedRepositoryMa
@After
public void tearDown() {
- ((RawLuceneRepositoryManager) repoManager).close();
+ repoManager.close();
}
- protected void createIndexAndRepoManager() {
- LuceneServiceImpl.luceneIndexFactory = new LuceneRawIndexFactory();
+ protected void createIndexAndRepoManager() throws IOException {
+ String luceneFolderPath = temporaryFolder.newFolder("lucene").getPath();
+ LuceneServiceImpl.luceneIndexFactory = new LuceneRawIndexFactory(luceneFolderPath);
indexStats = Mockito.mock(LuceneIndexStats.class);
indexForPR = Mockito.mock(LuceneRawIndex.class);
@@ -66,7 +69,8 @@ public class RawLuceneRepositoryManagerJUnitTest extends PartitionedRepositoryMa
when(indexForPR.getCache()).thenReturn(cache);
when(indexForPR.getRegionPath()).thenReturn("/testRegion");
when(indexForPR.withPersistence()).thenReturn(true);
- repoManager = new RawLuceneRepositoryManager(indexForPR, serializer);
+ repoManager = new RawLuceneRepositoryManager(indexForPR, serializer,
+ Executors.newSingleThreadExecutor(), luceneFolderPath);
repoManager.setUserRegionForRepositoryManager(userRegion);
repoManager.allowRepositoryComputation();
}
@@ -78,14 +82,14 @@ public class RawLuceneRepositoryManagerJUnitTest extends PartitionedRepositoryMa
}
@Override
- protected void checkRepository(IndexRepositoryImpl repo0, int bucketId) {
+ protected void checkRepositoryContainsBucket(IndexRepositoryImpl repo0, int... bucketIds) {
IndexWriter writer0 = repo0.getWriter();
Directory dir0 = writer0.getDirectory();
assertTrue(dir0 instanceof NIOFSDirectory);
}
@Override
- protected BucketRegion setUpMockBucket(int id) throws BucketNotFoundException {
+ protected void setUpMockBucket(int id) {
BucketRegion mockBucket = Mockito.mock(BucketRegion.class);
when(mockBucket.getId()).thenReturn(id);
when(userRegion.getBucketRegion(eq(id), eq(null))).thenReturn(mockBucket);
@@ -95,7 +99,6 @@ public class RawLuceneRepositoryManagerJUnitTest extends PartitionedRepositoryMa
dataBuckets.put(id, mockBucket);
repoManager.computeRepository(mockBucket.getId());
- return mockBucket;
}
@Test
--
To stop receiving notification emails like this one, please contact
udo@apache.org.
[geode] 03/05: Cosmetic
Posted by ud...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
udo pushed a commit to branch feature/GEODE-3926_3
in repository https://gitbox.apache.org/repos/asf/geode.git
commit b9d36b18341621e08c9b3fb6e675a39592c313eb
Author: Udo Kohlmeyer <uk...@pivotal.io>
AuthorDate: Thu Mar 29 15:32:04 2018 -0700
Cosmetic
---
.../internal/PartitionedRepositoryManager.java | 29 +++++++++++-----------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
index 92b4c08..092b332 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
@@ -88,20 +88,21 @@ public class PartitionedRepositoryManager implements RepositoryManager {
throw new BucketNotFoundException(
"User bucket was not found for region " + region + "bucket id " + bucketId);
} else {
- if (index.isIndexAvailable(userBucket.getId())) {
- repos.add(getRepository(userBucket.getId()));
- } else {
- waitingThreadPoolFromDM.execute(() -> {
- try {
- IndexRepository repository = getRepository(userBucket.getId());
- repos.add(repository);
- } catch (BucketNotFoundException e) {
- logger.debug(
- "Lucene Index creation in progress. Catching BucketNotFoundException");
- }
- });
- throw new LuceneIndexCreationInProgressException(
- "Lucene Index creation in progress for bucket: " + userBucket.getId());
+ if (LuceneServiceImpl.LUCENE_REINDEX) {
+ if (index.isIndexAvailable(userBucket.getId())) {
+ repos.add(getRepository(userBucket.getId()));
+ } else {
+ waitingThreadPoolFromDM.execute(() -> {
+ try {
+ IndexRepository repository = getRepository(userBucket.getId());
+ repos.add(repository);
+ } catch (BucketNotFoundException e) {
+ logger.debug("Lucene Index creation in progress. Catching BucketNotFoundException");
+ }
+ });
+ throw new LuceneIndexCreationInProgressException(
+ "Lucene Index creation in progress for bucket: " + userBucket.getId());
+ }
}
}
}
--
To stop receiving notification emails like this one, please contact
udo@apache.org.
[geode] 04/05: Added repos.add back into else statement
Posted by ud...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
udo pushed a commit to branch feature/GEODE-3926_3
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 9c8f726f2e03c28cc08fde9b259af1a1838bbde1
Author: Udo Kohlmeyer <uk...@pivotal.io>
AuthorDate: Thu Mar 29 16:30:41 2018 -0700
Added repos.add back into else statement
---
.../geode/cache/lucene/internal/PartitionedRepositoryManager.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
index 092b332..a3a74e6 100755
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java
@@ -104,6 +104,9 @@ public class PartitionedRepositoryManager implements RepositoryManager {
"Lucene Index creation in progress for bucket: " + userBucket.getId());
}
}
+ else{
+ repos.add(getRepository(userBucket.getId()));
+ }
}
}
--
To stop receiving notification emails like this one, please contact
udo@apache.org.
[geode] 05/05: Amended PartitionedRepositoryManagerJUnitTest.java
test
Posted by ud...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
udo pushed a commit to branch feature/GEODE-3926_3
in repository https://gitbox.apache.org/repos/asf/geode.git
commit cb5b44f70d838955e5ef9fdc4e1538f21f9edb71
Author: Udo Kohlmeyer <uk...@pivotal.io>
AuthorDate: Fri Mar 30 10:35:22 2018 -0700
Amended PartitionedRepositoryManagerJUnitTest.java test
---
.../internal/PartitionedRepositoryManagerJUnitTest.java | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java
index 38319b6..17cb061 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManagerJUnitTest.java
@@ -60,6 +60,7 @@ import org.apache.geode.cache.lucene.internal.partition.BucketTargetingMap;
import org.apache.geode.cache.lucene.internal.repository.IndexRepository;
import org.apache.geode.cache.lucene.internal.repository.IndexRepositoryImpl;
import org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
+import org.apache.geode.distributed.internal.DistributionConfig;
import org.apache.geode.distributed.internal.locks.DLockService;
import org.apache.geode.internal.cache.BucketAdvisor;
import org.apache.geode.internal.cache.BucketNotFoundException;
@@ -105,6 +106,7 @@ public class PartitionedRepositoryManagerJUnitTest {
@Before
public void setUp() throws IOException {
+ System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "luceneReindex","true");
cache = Fakes.cache();
userRegion = Mockito.mock(PartitionedRegion.class);
userDataStore = Mockito.mock(PartitionedRegionDataStore.class);
@@ -247,6 +249,18 @@ public class PartitionedRepositoryManagerJUnitTest {
repoManager.getRepositories(ctx);
}
+ @Test(expected = LuceneIndexCreationInProgressException.class)
+ public void queryByRegionFailingWithInProgressException()
+ throws LuceneIndexCreationInProgressException, BucketNotFoundException {
+ setUpMockBucket(0);
+ setUpMockBucket(1);
+
+ Set<Integer> buckets = new LinkedHashSet<>(Arrays.asList(0, 1));
+ InternalRegionFunctionContext ctx = Mockito.mock(InternalRegionFunctionContext.class);
+ when(ctx.getLocalBucketSet((any()))).thenReturn(buckets);
+ repoManager.getRepositories(ctx);
+ }
+
protected void checkRepositoryContainsBucket(IndexRepositoryImpl repo0, int... bucketIds) {
IndexWriter writer0 = repo0.getWriter();
RegionDirectory dir0 = (RegionDirectory) writer0.getDirectory();
--
To stop receiving notification emails like this one, please contact
udo@apache.org.
[geode] 02/05: Some cosmetic changes
Posted by ud...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
udo pushed a commit to branch feature/GEODE-3926_3
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 660e937e262d8ef4a05ee19e68a682ae33a09144
Author: Udo Kohlmeyer <uk...@pivotal.io>
AuthorDate: Thu Mar 29 15:22:16 2018 -0700
Some cosmetic changes
---
.../internal/ClusterDistributionManager.java | 21 +++++++++------------
.../cache/lucene/internal/LuceneServiceImpl.java | 4 ++--
2 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/ClusterDistributionManager.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/ClusterDistributionManager.java
index f190e72..6151c7f 100644
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/ClusterDistributionManager.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/ClusterDistributionManager.java
@@ -892,18 +892,15 @@ public class ClusterDistributionManager implements DistributionManager {
@Override
public Thread newThread(final Runnable command) {
ClusterDistributionManager.this.stats.incWaitingThreadStarts();// will it be ok?
- final Runnable r = new Runnable() {
- @Override
- public void run() {
- ClusterDistributionManager.this.stats.incWaitingThreads(1);// will it be ok
- try {
- ConnectionTable.threadWantsSharedResources();
- Connection.makeReaderThread();
- runUntilShutdown(command);
- } finally {
- ConnectionTable.releaseThreadsSockets();
- ClusterDistributionManager.this.stats.incWaitingThreads(-1);
- }
+ final Runnable r = () -> {
+ ClusterDistributionManager.this.stats.incWaitingThreads(1);// will it be ok
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ ClusterDistributionManager.this.stats.incWaitingThreads(-1);
}
};
Thread thread = new Thread(group, r,
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
index 3e08abd..1aa50e8 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
@@ -363,7 +363,7 @@ public class LuceneServiceImpl implements InternalLuceneService {
}
public List<LuceneIndex> getIndexes(String regionPath) {
- List<LuceneIndex> indexes = new ArrayList();
+ List<LuceneIndex> indexes = new ArrayList<>();
for (LuceneIndex index : getAllIndexes()) {
if (index.getRegionPath().equals(regionPath)) {
indexes.add(index);
@@ -373,7 +373,7 @@ public class LuceneServiceImpl implements InternalLuceneService {
}
public List<LuceneIndexCreationProfile> getDefinedIndexes(String regionPath) {
- List<LuceneIndexCreationProfile> profiles = new ArrayList();
+ List<LuceneIndexCreationProfile> profiles = new ArrayList<>();
for (LuceneIndexCreationProfile profile : getAllDefinedIndexes()) {
if (profile.getRegionPath().equals(regionPath)) {
profiles.add(profile);
--
To stop receiving notification emails like this one, please contact
udo@apache.org.