You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by nn...@apache.org on 2018/05/09 16:08:38 UTC
[geode] branch develop updated: GEODE-5109: Overriding the missing
profile check for Lucene (#1922)
This is an automated email from the ASF dual-hosted git repository.
nnag pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new d038373 GEODE-5109: Overriding the missing profile check for Lucene (#1922)
d038373 is described below
commit d0383734d7a8655b85947cf24596a800fe9d555a
Author: Nabarun Nag <na...@users.noreply.github.com>
AuthorDate: Wed May 9 09:08:30 2018 -0700
GEODE-5109: Overriding the missing profile check for Lucene (#1922)
* A new createRegionMessage created for Lucene reindex validation
* The check method is overridden to ignore missing profile check in case of reindex.
* A member may be missing a reindex operation because it is waiting for the index command, but this should not prevent another member from creating a Lucene index via reindex
* Backwards compatible because this message will not be sent to members older than 1.7
---
.../geode/internal/DataSerializableFixedID.java | 1 +
.../internal/cache/CreateRegionProcessor.java | 9 +-
.../apache/geode/internal/cache/LocalRegion.java | 6 +-
.../geode/internal/i18n/LocalizedStrings.java | 4 +-
.../internal/CreateRegionProcessorForLucene.java | 73 +++++++++
.../cache/lucene/internal/LuceneServiceImpl.java | 23 ++-
.../lucene/LuceneQueriesReindexDUnitTest.java | 182 +++++++++++++++++++++
.../internal/LuceneServiceImplJUnitTest.java | 5 +
8 files changed, 297 insertions(+), 6 deletions(-)
diff --git a/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java b/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java
index b5b6b5e..ecf20ab 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java
@@ -71,6 +71,7 @@ public interface DataSerializableFixedID extends SerializationVersions {
*
* In DSFIDFactory, add a case for the new class case FOO: return new FOO(in);
*/
+ short CREATE_REGION_MESSAGE_LUCENE = -159;
short FINAL_CHECK_PASSED_MESSAGE = -158;
short NETWORK_PARTITION_MESSAGE = -157;
short SUSPECT_MEMBERS_MESSAGE = -156;
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java b/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java
index ec55171..3ffb12e 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java
@@ -599,7 +599,7 @@ public class CreateRegionProcessor implements ProfileExchangeProcessor {
for (CacheServiceProfile remoteProfile : profile.cacheServiceProfiles) {
CacheServiceProfile localProfile = myProfiles.get(remoteProfile.getId());
if (localProfile == null) {
- cspResult = remoteProfile.getMissingProfileMessage(true);
+ cspResult = getMissingProfileMessage(remoteProfile, true);
} else {
cspResult = remoteProfile.checkCompatibility(rgn.getFullPath(), localProfile);
}
@@ -617,7 +617,7 @@ public class CreateRegionProcessor implements ProfileExchangeProcessor {
for (CacheServiceProfile localProfile : myProfiles.values()) {
if (!profile.cacheServiceProfiles.stream()
.anyMatch(remoteProfile -> remoteProfile.getId().equals(localProfile.getId()))) {
- cspResult = localProfile.getMissingProfileMessage(false);
+ cspResult = getMissingProfileMessage(localProfile, false);
break;
}
}
@@ -639,6 +639,11 @@ public class CreateRegionProcessor implements ProfileExchangeProcessor {
return result;
}
+ protected String getMissingProfileMessage(CacheServiceProfile profile,
+ boolean existsInThisMember) {
+ return profile.getMissingProfileMessage(existsInThisMember);
+ }
+
/**
* When many members are started concurrently, it is possible that an accessor or non-version
* generating replicate receives CreateRegionMessage before it is initialized, thus preventing
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
index c099160..a02874f 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
@@ -10429,6 +10429,10 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
return this.isUsedForParallelGatewaySenderQueue;
}
+ public void removeCacheServiceProfile(String profileID) {
+ this.cacheServiceProfiles.remove(profileID);
+ }
+
public AbstractGatewaySender getSerialGatewaySender() {
return this.serialGatewaySender;
}
@@ -10483,7 +10487,7 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
|| isUsedForPartitionedRegionBucket();
}
- Map<String, CacheServiceProfile> getCacheServiceProfiles() {
+ public Map<String, CacheServiceProfile> getCacheServiceProfiles() {
return this.cacheServiceProfiles.getSnapshot();
}
diff --git a/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java b/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java
index ff56a07..3552018 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java
@@ -7713,7 +7713,9 @@ public class LocalizedStrings {
new StringId(6665, "{0}: Enqueueing synchronization event: {1}");
public static final StringId PoolImpl_CACHE_MUST_BE_CREATED_BEFORE_CREATING_POOL =
new StringId(6666, "Cache must be created before creating pool");
-
+ public static final StringId LuceneIndexCreation_INDEX_CANNOT_BE_CREATED_DUE_TO_PROFILE_VIOLATION =
+ new StringId(6667,
+ "Lucene index {0} cannot be created because its parameters are incompatible with another Lucene index");
/** Testing strings, messageId 90000-99999 **/
/**
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/CreateRegionProcessorForLucene.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/CreateRegionProcessorForLucene.java
new file mode 100644
index 0000000..152c69c
--- /dev/null
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/CreateRegionProcessorForLucene.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.cache.lucene.internal;
+
+import java.util.Set;
+
+import org.apache.geode.distributed.internal.ReplyProcessor21;
+import org.apache.geode.internal.cache.CacheDistributionAdvisee;
+import org.apache.geode.internal.cache.CacheDistributionAdvisor;
+import org.apache.geode.internal.cache.CacheServiceProfile;
+import org.apache.geode.internal.cache.CreateRegionProcessor;
+
+public class CreateRegionProcessorForLucene extends CreateRegionProcessor {
+
+ public CreateRegionProcessorForLucene(CacheDistributionAdvisee newRegion) {
+ super(newRegion);
+ }
+
+ @Override
+ protected CreateRegionMessage getCreateRegionMessage(Set recps, ReplyProcessor21 proc,
+ boolean useMcast) {
+ CreateRegionMessage msg = new CreateRegionMessage(this.newRegion.getFullPath(),
+ (CacheDistributionAdvisor.CacheProfile) this.newRegion.getProfile(), proc.getProcessorId());
+ msg.concurrencyChecksEnabled = this.newRegion.getAttributes().getConcurrencyChecksEnabled();
+ msg.setMulticast(useMcast);
+ msg.setRecipients(recps);
+ return msg;
+
+ }
+
+ public static class CreateRegionMessage extends CreateRegionProcessor.CreateRegionMessage {
+ public CreateRegionMessage(String regionPath,
+ CacheDistributionAdvisor.CacheProfile cacheProfile, int processorId) {
+ this.regionPath = regionPath;
+ this.profile = cacheProfile;
+ this.processorId = processorId;
+ }
+
+ public CreateRegionMessage() {}
+
+ @Override
+ public int getDSFID() {
+ return CREATE_REGION_MESSAGE_LUCENE;
+ }
+
+
+ @Override
+ protected String getMissingProfileMessage(CacheServiceProfile profile,
+ boolean existsInThisMember) {
+ /*
+ * Reindex operation
+ * Null string return is treated as no error scenario
+ * Since this is a reindex operation it is acceptable for a region to have a missing cache
+ * service profile. It will eventually receive a create index command
+ */
+
+ return null;
+ }
+ }
+
+}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
index 211b508..5267e30 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
@@ -15,6 +15,8 @@
package org.apache.geode.cache.lucene.internal;
+import static org.apache.geode.internal.DataSerializableFixedID.CREATE_REGION_MESSAGE_LUCENE;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -244,9 +246,20 @@ public class LuceneServiceImpl implements InternalLuceneService {
LuceneSerializer serializer) {
validateRegionAttributes(region.getAttributes());
- region.addCacheServiceProfile(new LuceneIndexCreationProfile(indexName, regionPath, fields,
- analyzer, fieldAnalyzers, serializer));
+ LuceneIndexCreationProfile luceneIndexCreationProfile = new LuceneIndexCreationProfile(
+ indexName, regionPath, fields, analyzer, fieldAnalyzers, serializer);
+
+ region.addCacheServiceProfile(luceneIndexCreationProfile);
+ try {
+ validateLuceneIndexProfile(region);
+ } catch (Exception e) {
+ region.removeCacheServiceProfile(luceneIndexCreationProfile.getId());
+ throw new UnsupportedOperationException(
+ LocalizedStrings.LuceneIndexCreation_INDEX_CANNOT_BE_CREATED_DUE_TO_PROFILE_VIOLATION
+ .toString(indexName),
+ e);
+ }
String aeqId = LuceneServiceImpl.getUniqueIndexName(indexName, regionPath);
region.updatePRConfigWithNewGatewaySender(aeqId);
LuceneIndexImpl luceneIndex = beforeDataRegionCreated(indexName, regionPath,
@@ -257,6 +270,10 @@ public class LuceneServiceImpl implements InternalLuceneService {
createLuceneIndexOnDataRegion(region, luceneIndex);
}
+ protected void validateLuceneIndexProfile(PartitionedRegion region) {
+ new CreateRegionProcessorForLucene(region).initializeRegion();
+ }
+
protected boolean createLuceneIndexOnDataRegion(final PartitionedRegion userRegion,
final InternalLuceneIndex luceneIndex) {
try {
@@ -555,6 +572,8 @@ public class LuceneServiceImpl implements InternalLuceneService {
/** Public for test purposes */
public static void registerDataSerializables() {
+ DSFIDFactory.registerDSFID(CREATE_REGION_MESSAGE_LUCENE,
+ CreateRegionProcessorForLucene.CreateRegionMessage.class);
DSFIDFactory.registerDSFID(DataSerializableFixedID.LUCENE_CHUNK_KEY, ChunkKey.class);
DSFIDFactory.registerDSFID(DataSerializableFixedID.LUCENE_FILE, File.class);
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java
index e25ca9a..f6dcef4 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java
@@ -18,13 +18,17 @@ import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.INDEX_NAME;
import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.REGION_NAME;
import static org.junit.Assert.assertTrue;
+import java.util.concurrent.TimeUnit;
+
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;
+import org.awaitility.Awaitility;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.apache.geode.cache.lucene.internal.LuceneIndexFactoryImpl;
+import org.apache.geode.internal.cache.PartitionedRegion;
import org.apache.geode.test.dunit.AsyncInvocation;
import org.apache.geode.test.dunit.SerializableRunnableIF;
import org.apache.geode.test.junit.categories.DistributedTest;
@@ -92,6 +96,184 @@ public class LuceneQueriesReindexDUnitTest extends LuceneQueriesAccessorBase {
@Test
@Parameters(method = "getListOfRegionTestTypes")
+ public void dropAndRecreateIndexWithDifferentFieldsShouldFail(RegionTestableType regionTestType) {
+ SerializableRunnableIF createIndex = () -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ luceneService.createIndexFactory().addField("text").create(INDEX_NAME, REGION_NAME);
+ };
+ dataStore1.invoke(() -> initDataStore(createIndex, regionTestType));
+ dataStore2.invoke(() -> initDataStore(createIndex, regionTestType));
+ accessor.invoke(() -> initAccessor(createIndex, regionTestType));
+
+ putDataInRegion(accessor);
+ assertTrue(waitForFlushBeforeExecuteTextSearch(accessor, 60000));
+ assertTrue(waitForFlushBeforeExecuteTextSearch(dataStore1, 60000));
+ executeTextSearch(accessor);
+
+ dataStore1.invoke(() -> destroyIndex());
+
+
+ // re-index stored data
+ AsyncInvocation ai1 = dataStore1.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+ indexFactory.create(INDEX_NAME, REGION_NAME, true);
+ });
+
+ AsyncInvocation ai2 = dataStore2.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+ indexFactory.create(INDEX_NAME, REGION_NAME, true);
+ });
+
+ waitAndVerifyThatIndexCreationFails(ai1, ai2);
+
+ }
+
+ @Test
+ @Parameters(method = "getListOfRegionTestTypes")
+ public void recreateIndexWithDifferentFieldsShouldFail(RegionTestableType regionTestType) {
+
+ dataStore1.invoke(() -> {
+ regionTestType.createDataStore(getCache(), REGION_NAME);
+ });
+ dataStore2.invoke(() -> {
+ regionTestType.createDataStore(getCache(), REGION_NAME);
+ });
+ accessor.invoke(() -> {
+ regionTestType.createAccessor(getCache(), REGION_NAME);
+ });
+
+ putDataInRegion(accessor);
+
+ // re-index stored data
+ AsyncInvocation ai1 = dataStore1.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+ indexFactory.create(INDEX_NAME, REGION_NAME, true);
+ });
+
+ AsyncInvocation ai2 = dataStore2.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+ indexFactory.create(INDEX_NAME, REGION_NAME, true);
+ });
+
+ waitAndVerifyThatIndexCreationFails(ai1, ai2);
+
+ }
+
+ @Test
+ @Parameters(method = "getListOfRegionTestTypes")
+ public void recreateDifferentIndexesWithDifferentFieldsShouldPass(
+ RegionTestableType regionTestType) throws Exception {
+
+ dataStore1.invoke(() -> {
+ regionTestType.createDataStore(getCache(), REGION_NAME);
+ });
+ dataStore2.invoke(() -> {
+ regionTestType.createDataStore(getCache(), REGION_NAME);
+ });
+ accessor.invoke(() -> {
+ regionTestType.createAccessor(getCache(), REGION_NAME);
+ });
+
+ putDataInRegion(accessor);
+
+ // re-index stored data
+ AsyncInvocation ai1 = dataStore1.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+ indexFactory.create(INDEX_NAME, REGION_NAME, true);
+ });
+
+ AsyncInvocation ai2 = dataStore2.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+ indexFactory.create(INDEX_NAME + "2", REGION_NAME, true);
+ });
+
+ AsyncInvocation ai3 = dataStore1.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+ indexFactory.create(INDEX_NAME + "2", REGION_NAME, true);
+ });
+
+ AsyncInvocation ai4 = dataStore2.invokeAsync(() -> {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+ indexFactory.create(INDEX_NAME, REGION_NAME, true);
+ });
+
+ AsyncInvocation ai5 = accessor.invokeAsync(() -> {
+ if (getCache().getRegion(REGION_NAME) instanceof PartitionedRegion) {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+ indexFactory.create(INDEX_NAME + "1", REGION_NAME, true);
+ }
+ });
+
+ AsyncInvocation ai6 = accessor.invokeAsync(() -> {
+ if (getCache().getRegion(REGION_NAME) instanceof PartitionedRegion) {
+ LuceneService luceneService = LuceneServiceProvider.get(getCache());
+ LuceneIndexFactoryImpl indexFactory =
+ (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+ indexFactory.create(INDEX_NAME + "2", REGION_NAME, true);
+ }
+ });
+
+ ai1.join();
+ ai2.join();
+ ai3.join();
+ ai4.join();
+ ai5.join();
+ ai6.join();
+
+ ai1.checkException();
+ ai2.checkException();
+ ai3.checkException();
+ ai4.checkException();
+ ai5.checkException();
+ ai6.checkException();
+
+ waitForFlushBeforeExecuteTextSearch(accessor, 60000);
+ executeTextSearch(accessor);
+ }
+
+ private void waitAndVerifyThatIndexCreationFails(AsyncInvocation ai1, AsyncInvocation ai2) {
+ try {
+ Awaitility.await().atMost(1, TimeUnit.MINUTES).until(() -> {
+ boolean status1;
+ boolean status2;
+ try {
+ status1 = ai1.getException() instanceof UnsupportedOperationException;
+ assertTrue(status1);
+ } catch (AssertionError er) {
+ try {
+ status2 = ai2.getException() instanceof UnsupportedOperationException;
+ assertTrue(status2);
+ } catch (AssertionError er2) {
+ assertTrue(false);
+ }
+ }
+ });
+ } finally {
+ ai1.cancel(true);
+ ai2.cancel(true);
+ }
+ }
+
+ @Test
+ @Parameters(method = "getListOfRegionTestTypes")
public void reindexThenQuery(RegionTestableType regionTestType) throws Exception {
SerializableRunnableIF createIndex = () -> {
LuceneService luceneService = LuceneServiceProvider.get(getCache());
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
index 876811f..acc384c 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
@@ -165,6 +165,11 @@ public class LuceneServiceImplJUnitTest {
}
@Override
+ protected void validateLuceneIndexProfile(PartitionedRegion region) {
+
+ }
+
+ @Override
protected void validateAllMembersAreTheSameVersion(PartitionedRegion region) {
}
--
To stop receiving notification emails like this one, please contact
nnag@apache.org.