You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by nn...@apache.org on 2018/05/09 16:08:38 UTC

[geode] branch develop updated: GEODE-5109: Overriding the missing profile check for Lucene (#1922)

This is an automated email from the ASF dual-hosted git repository.

nnag pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git


The following commit(s) were added to refs/heads/develop by this push:
     new d038373  GEODE-5109: Overriding the missing profile check for Lucene (#1922)
d038373 is described below

commit d0383734d7a8655b85947cf24596a800fe9d555a
Author: Nabarun Nag <na...@users.noreply.github.com>
AuthorDate: Wed May 9 09:08:30 2018 -0700

    GEODE-5109: Overriding the missing profile check for Lucene (#1922)
    
            * A new createRegionMessage created for Lucene reindex validation
    	* The check method is overridden to ignore missing profile check in case of reindex.
    	* A member may be missing a reindex operation because it is waiting for the index command, but this should not prevent another member from creating a Lucene index via reindex
    	* Backwards compatible because this message will not be sent to members older than 1.7
---
 .../geode/internal/DataSerializableFixedID.java    |   1 +
 .../internal/cache/CreateRegionProcessor.java      |   9 +-
 .../apache/geode/internal/cache/LocalRegion.java   |   6 +-
 .../geode/internal/i18n/LocalizedStrings.java      |   4 +-
 .../internal/CreateRegionProcessorForLucene.java   |  73 +++++++++
 .../cache/lucene/internal/LuceneServiceImpl.java   |  23 ++-
 .../lucene/LuceneQueriesReindexDUnitTest.java      | 182 +++++++++++++++++++++
 .../internal/LuceneServiceImplJUnitTest.java       |   5 +
 8 files changed, 297 insertions(+), 6 deletions(-)

diff --git a/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java b/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java
index b5b6b5e..ecf20ab 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/DataSerializableFixedID.java
@@ -71,6 +71,7 @@ public interface DataSerializableFixedID extends SerializationVersions {
    *
    * In DSFIDFactory, add a case for the new class case FOO: return new FOO(in);
    */
+  short CREATE_REGION_MESSAGE_LUCENE = -159;
   short FINAL_CHECK_PASSED_MESSAGE = -158;
   short NETWORK_PARTITION_MESSAGE = -157;
   short SUSPECT_MEMBERS_MESSAGE = -156;
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java b/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java
index ec55171..3ffb12e 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/CreateRegionProcessor.java
@@ -599,7 +599,7 @@ public class CreateRegionProcessor implements ProfileExchangeProcessor {
       for (CacheServiceProfile remoteProfile : profile.cacheServiceProfiles) {
         CacheServiceProfile localProfile = myProfiles.get(remoteProfile.getId());
         if (localProfile == null) {
-          cspResult = remoteProfile.getMissingProfileMessage(true);
+          cspResult = getMissingProfileMessage(remoteProfile, true);
         } else {
           cspResult = remoteProfile.checkCompatibility(rgn.getFullPath(), localProfile);
         }
@@ -617,7 +617,7 @@ public class CreateRegionProcessor implements ProfileExchangeProcessor {
           for (CacheServiceProfile localProfile : myProfiles.values()) {
             if (!profile.cacheServiceProfiles.stream()
                 .anyMatch(remoteProfile -> remoteProfile.getId().equals(localProfile.getId()))) {
-              cspResult = localProfile.getMissingProfileMessage(false);
+              cspResult = getMissingProfileMessage(localProfile, false);
               break;
             }
           }
@@ -639,6 +639,11 @@ public class CreateRegionProcessor implements ProfileExchangeProcessor {
       return result;
     }
 
+    protected String getMissingProfileMessage(CacheServiceProfile profile,
+        boolean existsInThisMember) {
+      return profile.getMissingProfileMessage(existsInThisMember);
+    }
+
     /**
      * When many members are started concurrently, it is possible that an accessor or non-version
      * generating replicate receives CreateRegionMessage before it is initialized, thus preventing
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
index c099160..a02874f 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
@@ -10429,6 +10429,10 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
     return this.isUsedForParallelGatewaySenderQueue;
   }
 
+  public void removeCacheServiceProfile(String profileID) {
+    this.cacheServiceProfiles.remove(profileID);
+  }
+
   public AbstractGatewaySender getSerialGatewaySender() {
     return this.serialGatewaySender;
   }
@@ -10483,7 +10487,7 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
         || isUsedForPartitionedRegionBucket();
   }
 
-  Map<String, CacheServiceProfile> getCacheServiceProfiles() {
+  public Map<String, CacheServiceProfile> getCacheServiceProfiles() {
     return this.cacheServiceProfiles.getSnapshot();
   }
 
diff --git a/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java b/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java
index ff56a07..3552018 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/i18n/LocalizedStrings.java
@@ -7713,7 +7713,9 @@ public class LocalizedStrings {
       new StringId(6665, "{0}: Enqueueing synchronization event: {1}");
   public static final StringId PoolImpl_CACHE_MUST_BE_CREATED_BEFORE_CREATING_POOL =
       new StringId(6666, "Cache must be created before creating pool");
-
+  public static final StringId LuceneIndexCreation_INDEX_CANNOT_BE_CREATED_DUE_TO_PROFILE_VIOLATION =
+      new StringId(6667,
+          "Lucene index {0} cannot be created because its parameters are incompatible with another Lucene index");
   /** Testing strings, messageId 90000-99999 **/
 
   /**
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/CreateRegionProcessorForLucene.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/CreateRegionProcessorForLucene.java
new file mode 100644
index 0000000..152c69c
--- /dev/null
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/CreateRegionProcessorForLucene.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.cache.lucene.internal;
+
+import java.util.Set;
+
+import org.apache.geode.distributed.internal.ReplyProcessor21;
+import org.apache.geode.internal.cache.CacheDistributionAdvisee;
+import org.apache.geode.internal.cache.CacheDistributionAdvisor;
+import org.apache.geode.internal.cache.CacheServiceProfile;
+import org.apache.geode.internal.cache.CreateRegionProcessor;
+
+public class CreateRegionProcessorForLucene extends CreateRegionProcessor {
+
+  public CreateRegionProcessorForLucene(CacheDistributionAdvisee newRegion) {
+    super(newRegion);
+  }
+
+  @Override
+  protected CreateRegionMessage getCreateRegionMessage(Set recps, ReplyProcessor21 proc,
+      boolean useMcast) {
+    CreateRegionMessage msg = new CreateRegionMessage(this.newRegion.getFullPath(),
+        (CacheDistributionAdvisor.CacheProfile) this.newRegion.getProfile(), proc.getProcessorId());
+    msg.concurrencyChecksEnabled = this.newRegion.getAttributes().getConcurrencyChecksEnabled();
+    msg.setMulticast(useMcast);
+    msg.setRecipients(recps);
+    return msg;
+
+  }
+
+  public static class CreateRegionMessage extends CreateRegionProcessor.CreateRegionMessage {
+    public CreateRegionMessage(String regionPath,
+        CacheDistributionAdvisor.CacheProfile cacheProfile, int processorId) {
+      this.regionPath = regionPath;
+      this.profile = cacheProfile;
+      this.processorId = processorId;
+    }
+
+    public CreateRegionMessage() {}
+
+    @Override
+    public int getDSFID() {
+      return CREATE_REGION_MESSAGE_LUCENE;
+    }
+
+
+    @Override
+    protected String getMissingProfileMessage(CacheServiceProfile profile,
+        boolean existsInThisMember) {
+      /*
+       * Reindex operation
+       * Null string return is treated as no error scenario
+       * Since this is a reindex operation it is acceptable for a region to have a missing cache
+       * service profile. It will eventually receive a create index command
+       */
+
+      return null;
+    }
+  }
+
+}
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
index 211b508..5267e30 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java
@@ -15,6 +15,8 @@
 
 package org.apache.geode.cache.lucene.internal;
 
+import static org.apache.geode.internal.DataSerializableFixedID.CREATE_REGION_MESSAGE_LUCENE;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -244,9 +246,20 @@ public class LuceneServiceImpl implements InternalLuceneService {
       LuceneSerializer serializer) {
     validateRegionAttributes(region.getAttributes());
 
-    region.addCacheServiceProfile(new LuceneIndexCreationProfile(indexName, regionPath, fields,
-        analyzer, fieldAnalyzers, serializer));
+    LuceneIndexCreationProfile luceneIndexCreationProfile = new LuceneIndexCreationProfile(
+        indexName, regionPath, fields, analyzer, fieldAnalyzers, serializer);
+
+    region.addCacheServiceProfile(luceneIndexCreationProfile);
 
+    try {
+      validateLuceneIndexProfile(region);
+    } catch (Exception e) {
+      region.removeCacheServiceProfile(luceneIndexCreationProfile.getId());
+      throw new UnsupportedOperationException(
+          LocalizedStrings.LuceneIndexCreation_INDEX_CANNOT_BE_CREATED_DUE_TO_PROFILE_VIOLATION
+              .toString(indexName),
+          e);
+    }
     String aeqId = LuceneServiceImpl.getUniqueIndexName(indexName, regionPath);
     region.updatePRConfigWithNewGatewaySender(aeqId);
     LuceneIndexImpl luceneIndex = beforeDataRegionCreated(indexName, regionPath,
@@ -257,6 +270,10 @@ public class LuceneServiceImpl implements InternalLuceneService {
     createLuceneIndexOnDataRegion(region, luceneIndex);
   }
 
+  protected void validateLuceneIndexProfile(PartitionedRegion region) {
+    new CreateRegionProcessorForLucene(region).initializeRegion();
+  }
+
   protected boolean createLuceneIndexOnDataRegion(final PartitionedRegion userRegion,
       final InternalLuceneIndex luceneIndex) {
     try {
@@ -555,6 +572,8 @@ public class LuceneServiceImpl implements InternalLuceneService {
 
   /** Public for test purposes */
   public static void registerDataSerializables() {
+    DSFIDFactory.registerDSFID(CREATE_REGION_MESSAGE_LUCENE,
+        CreateRegionProcessorForLucene.CreateRegionMessage.class);
     DSFIDFactory.registerDSFID(DataSerializableFixedID.LUCENE_CHUNK_KEY, ChunkKey.class);
 
     DSFIDFactory.registerDSFID(DataSerializableFixedID.LUCENE_FILE, File.class);
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java
index e25ca9a..f6dcef4 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesReindexDUnitTest.java
@@ -18,13 +18,17 @@ import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.INDEX_NAME;
 import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.REGION_NAME;
 import static org.junit.Assert.assertTrue;
 
+import java.util.concurrent.TimeUnit;
+
 import junitparams.JUnitParamsRunner;
 import junitparams.Parameters;
+import org.awaitility.Awaitility;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 
 import org.apache.geode.cache.lucene.internal.LuceneIndexFactoryImpl;
+import org.apache.geode.internal.cache.PartitionedRegion;
 import org.apache.geode.test.dunit.AsyncInvocation;
 import org.apache.geode.test.dunit.SerializableRunnableIF;
 import org.apache.geode.test.junit.categories.DistributedTest;
@@ -92,6 +96,184 @@ public class LuceneQueriesReindexDUnitTest extends LuceneQueriesAccessorBase {
 
   @Test
   @Parameters(method = "getListOfRegionTestTypes")
+  public void dropAndRecreateIndexWithDifferentFieldsShouldFail(RegionTestableType regionTestType) {
+    SerializableRunnableIF createIndex = () -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      luceneService.createIndexFactory().addField("text").create(INDEX_NAME, REGION_NAME);
+    };
+    dataStore1.invoke(() -> initDataStore(createIndex, regionTestType));
+    dataStore2.invoke(() -> initDataStore(createIndex, regionTestType));
+    accessor.invoke(() -> initAccessor(createIndex, regionTestType));
+
+    putDataInRegion(accessor);
+    assertTrue(waitForFlushBeforeExecuteTextSearch(accessor, 60000));
+    assertTrue(waitForFlushBeforeExecuteTextSearch(dataStore1, 60000));
+    executeTextSearch(accessor);
+
+    dataStore1.invoke(() -> destroyIndex());
+
+
+    // re-index stored data
+    AsyncInvocation ai1 = dataStore1.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+      indexFactory.create(INDEX_NAME, REGION_NAME, true);
+    });
+
+    AsyncInvocation ai2 = dataStore2.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+      indexFactory.create(INDEX_NAME, REGION_NAME, true);
+    });
+
+    waitAndVerifyThatIndexCreationFails(ai1, ai2);
+
+  }
+
+  @Test
+  @Parameters(method = "getListOfRegionTestTypes")
+  public void recreateIndexWithDifferentFieldsShouldFail(RegionTestableType regionTestType) {
+
+    dataStore1.invoke(() -> {
+      regionTestType.createDataStore(getCache(), REGION_NAME);
+    });
+    dataStore2.invoke(() -> {
+      regionTestType.createDataStore(getCache(), REGION_NAME);
+    });
+    accessor.invoke(() -> {
+      regionTestType.createAccessor(getCache(), REGION_NAME);
+    });
+
+    putDataInRegion(accessor);
+
+    // re-index stored data
+    AsyncInvocation ai1 = dataStore1.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+      indexFactory.create(INDEX_NAME, REGION_NAME, true);
+    });
+
+    AsyncInvocation ai2 = dataStore2.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+      indexFactory.create(INDEX_NAME, REGION_NAME, true);
+    });
+
+    waitAndVerifyThatIndexCreationFails(ai1, ai2);
+
+  }
+
+  @Test
+  @Parameters(method = "getListOfRegionTestTypes")
+  public void recreateDifferentIndexesWithDifferentFieldsShouldPass(
+      RegionTestableType regionTestType) throws Exception {
+
+    dataStore1.invoke(() -> {
+      regionTestType.createDataStore(getCache(), REGION_NAME);
+    });
+    dataStore2.invoke(() -> {
+      regionTestType.createDataStore(getCache(), REGION_NAME);
+    });
+    accessor.invoke(() -> {
+      regionTestType.createAccessor(getCache(), REGION_NAME);
+    });
+
+    putDataInRegion(accessor);
+
+    // re-index stored data
+    AsyncInvocation ai1 = dataStore1.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+      indexFactory.create(INDEX_NAME, REGION_NAME, true);
+    });
+
+    AsyncInvocation ai2 = dataStore2.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+      indexFactory.create(INDEX_NAME + "2", REGION_NAME, true);
+    });
+
+    AsyncInvocation ai3 = dataStore1.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+      indexFactory.create(INDEX_NAME + "2", REGION_NAME, true);
+    });
+
+    AsyncInvocation ai4 = dataStore2.invokeAsync(() -> {
+      LuceneService luceneService = LuceneServiceProvider.get(getCache());
+      LuceneIndexFactoryImpl indexFactory =
+          (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+      indexFactory.create(INDEX_NAME, REGION_NAME, true);
+    });
+
+    AsyncInvocation ai5 = accessor.invokeAsync(() -> {
+      if (getCache().getRegion(REGION_NAME) instanceof PartitionedRegion) {
+        LuceneService luceneService = LuceneServiceProvider.get(getCache());
+        LuceneIndexFactoryImpl indexFactory =
+            (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text");
+        indexFactory.create(INDEX_NAME + "1", REGION_NAME, true);
+      }
+    });
+
+    AsyncInvocation ai6 = accessor.invokeAsync(() -> {
+      if (getCache().getRegion(REGION_NAME) instanceof PartitionedRegion) {
+        LuceneService luceneService = LuceneServiceProvider.get(getCache());
+        LuceneIndexFactoryImpl indexFactory =
+            (LuceneIndexFactoryImpl) luceneService.createIndexFactory().addField("text2");
+        indexFactory.create(INDEX_NAME + "2", REGION_NAME, true);
+      }
+    });
+
+    ai1.join();
+    ai2.join();
+    ai3.join();
+    ai4.join();
+    ai5.join();
+    ai6.join();
+
+    ai1.checkException();
+    ai2.checkException();
+    ai3.checkException();
+    ai4.checkException();
+    ai5.checkException();
+    ai6.checkException();
+
+    waitForFlushBeforeExecuteTextSearch(accessor, 60000);
+    executeTextSearch(accessor);
+  }
+
+  private void waitAndVerifyThatIndexCreationFails(AsyncInvocation ai1, AsyncInvocation ai2) {
+    try {
+      Awaitility.await().atMost(1, TimeUnit.MINUTES).until(() -> {
+        boolean status1;
+        boolean status2;
+        try {
+          status1 = ai1.getException() instanceof UnsupportedOperationException;
+          assertTrue(status1);
+        } catch (AssertionError er) {
+          try {
+            status2 = ai2.getException() instanceof UnsupportedOperationException;
+            assertTrue(status2);
+          } catch (AssertionError er2) {
+            assertTrue(false);
+          }
+        }
+      });
+    } finally {
+      ai1.cancel(true);
+      ai2.cancel(true);
+    }
+  }
+
+  @Test
+  @Parameters(method = "getListOfRegionTestTypes")
   public void reindexThenQuery(RegionTestableType regionTestType) throws Exception {
     SerializableRunnableIF createIndex = () -> {
       LuceneService luceneService = LuceneServiceProvider.get(getCache());
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
index 876811f..acc384c 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneServiceImplJUnitTest.java
@@ -165,6 +165,11 @@ public class LuceneServiceImplJUnitTest {
     }
 
     @Override
+    protected void validateLuceneIndexProfile(PartitionedRegion region) {
+
+    }
+
+    @Override
     protected void validateAllMembersAreTheSameVersion(PartitionedRegion region) {
 
     }

-- 
To stop receiving notification emails like this one, please contact
nnag@apache.org.