You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by nn...@apache.org on 2017/01/16 17:54:57 UTC
geode git commit: GEODE-1733: Lucene stats are displayed correctly.
Repository: geode
Updated Branches:
refs/heads/develop a4f38f33a -> f1cebc2f0
GEODE-1733: Lucene stats are displayed correctly.
* Lucene stats are displayed correctly when rebalancing occurs or a server is reinstantiated
* Listener added which creates the LuceneIndexRepository when a bucket becomes primary.
* LuceneIndexRepository no longer being created lazily.
Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/f1cebc2f
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/f1cebc2f
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/f1cebc2f
Branch: refs/heads/develop
Commit: f1cebc2f085807d48adf58d7ad0fbebced49e043
Parents: a4f38f3
Author: nabarun <nn...@pivotal.io>
Authored: Wed Jan 4 18:07:03 2017 -0800
Committer: nabarun <nn...@pivotal.io>
Committed: Mon Jan 16 09:51:42 2017 -0800
----------------------------------------------------------------------
.../LuceneIndexForPartitionedRegion.java | 27 ++++++++----
.../internal/LucenePrimaryBucketListener.java | 43 ++++++++++++++++++++
.../repository/IndexRepositoryImpl.java | 3 +-
.../LuceneIndexForPartitionedRegionTest.java | 18 ++++----
.../IndexRepositoryImplJUnitTest.java | 9 +++-
5 files changed, 81 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
index c760726..5968912 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
@@ -33,6 +33,7 @@ import org.apache.geode.cache.lucene.internal.filesystem.File;
import org.apache.geode.cache.lucene.internal.filesystem.FileSystemStats;
import org.apache.geode.cache.lucene.internal.repository.RepositoryManager;
import org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
+import org.apache.geode.cache.partition.PartitionListener;
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.cache.PartitionedRegion;
@@ -78,17 +79,22 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
// create PR chunkRegion, but not to create its buckets for now
final String chunkRegionName = createChunkRegionName();
+
+ // we will create RegionDirectories on the fly when data comes in
+ HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(getFieldNames());
+ PartitionedRepositoryManager partitionedRepositoryManager =
+ new PartitionedRepositoryManager(this, mapper);
+ LucenePrimaryBucketListener lucenePrimaryBucketListener =
+ new LucenePrimaryBucketListener(partitionedRepositoryManager);
if (!chunkRegionExists(chunkRegionName)) {
chunkRegion = createChunkRegion(regionShortCut, fileRegionName, partitionAttributes,
- chunkRegionName, regionAttributes);
+ chunkRegionName, regionAttributes, lucenePrimaryBucketListener);
}
fileSystemStats.setFileSupplier(() -> (int) getFileRegion().getLocalSize());
fileSystemStats.setChunkSupplier(() -> (int) getChunkRegion().getLocalSize());
fileSystemStats.setBytesSupplier(() -> getChunkRegion().getPrStats().getDataStoreBytesInUse());
- // we will create RegionDirectories on the fly when data comes in
- HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(getFieldNames());
- return new PartitionedRepositoryManager(this, mapper);
+ return partitionedRepositoryManager;
}
public PartitionedRegion getFileRegion() {
@@ -110,7 +116,7 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
Region createFileRegion(final RegionShortcut regionShortCut, final String fileRegionName,
final PartitionAttributes partitionAttributes, final RegionAttributes regionAttributes) {
return createRegion(fileRegionName, regionShortCut, this.regionPath, partitionAttributes,
- regionAttributes);
+ regionAttributes, null);
}
public String createFileRegionName() {
@@ -123,9 +129,10 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
Region<ChunkKey, byte[]> createChunkRegion(final RegionShortcut regionShortCut,
final String fileRegionName, final PartitionAttributes partitionAttributes,
- final String chunkRegionName, final RegionAttributes regionAttributes) {
+ final String chunkRegionName, final RegionAttributes regionAttributes,
+ final PartitionListener lucenePrimaryBucketListener) {
return createRegion(chunkRegionName, regionShortCut, fileRegionName, partitionAttributes,
- regionAttributes);
+ regionAttributes, lucenePrimaryBucketListener);
}
public String createChunkRegionName() {
@@ -142,9 +149,13 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
protected <K, V> Region<K, V> createRegion(final String regionName,
final RegionShortcut regionShortCut, final String colocatedWithRegionName,
- final PartitionAttributes partitionAttributes, final RegionAttributes regionAttributes) {
+ final PartitionAttributes partitionAttributes, final RegionAttributes regionAttributes,
+ PartitionListener lucenePrimaryBucketListener) {
PartitionAttributesFactory partitionAttributesFactory =
new PartitionAttributesFactory<String, File>();
+ if (lucenePrimaryBucketListener != null) {
+ partitionAttributesFactory.addPartitionListener(lucenePrimaryBucketListener);
+ }
partitionAttributesFactory.setColocatedWith(colocatedWithRegionName);
configureLuceneRegionAttributesFactory(partitionAttributesFactory, partitionAttributes);
http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
new file mode 100644
index 0000000..386a8b0
--- /dev/null
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.cache.lucene.internal;
+
+import org.apache.geode.GemFireException;
+import org.apache.geode.cache.execute.FunctionException;
+import org.apache.geode.cache.partition.PartitionListenerAdapter;
+import org.apache.geode.internal.cache.BucketNotFoundException;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.logging.log4j.Logger;
+
+public class LucenePrimaryBucketListener extends PartitionListenerAdapter {
+ private static final Logger logger = LogService.getLogger();
+ private PartitionedRepositoryManager lucenePartitionRepositoryManager;
+
+ public LucenePrimaryBucketListener(PartitionedRepositoryManager partitionedRepositoryManager) {
+ lucenePartitionRepositoryManager = partitionedRepositoryManager;
+ }
+
+ @Override
+ public void afterPrimary(int bucketId) {
+ try {
+ lucenePartitionRepositoryManager.getRepository(bucketId);
+ } catch (BucketNotFoundException e) {
+ logger.warn(
+ "Index repository could not be created when index chunk region bucket became primary. "
+ + "Deferring index repository to be created lazily during lucene query execution."
+ + e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
index 572b5ea..4e86eb5 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
@@ -20,6 +20,7 @@ import org.apache.geode.cache.lucene.internal.LuceneIndexStats;
import org.apache.geode.cache.lucene.internal.repository.serializer.LuceneSerializer;
import org.apache.geode.cache.lucene.internal.repository.serializer.SerializerUtil;
import org.apache.geode.distributed.internal.DistributionConfig;
+import org.apache.geode.internal.cache.BucketRegion;
import org.apache.geode.internal.logging.LogService;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.document.Document;
@@ -163,7 +164,7 @@ public class IndexRepositoryImpl implements IndexRepository {
private class DocumentCountSupplier implements IntSupplier {
@Override
public int getAsInt() {
- if (isClosed()) {
+ if (isClosed() || !((BucketRegion) userRegion).getBucketAdvisor().isPrimary()) {
stats.removeDocumentsSupplier(this);
return 0;
}
http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
index 5c0c0a0..15235eb 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
@@ -245,7 +245,7 @@ public class LuceneIndexForPartitionedRegionTest {
index.setSearchableFields(new String[] {"field"});
LuceneIndexForPartitionedRegion spy = spy(index);
doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
- doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+ doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
doReturn(null).when(spy).createAEQ(eq(region));
spy.initialize();
return spy;
@@ -264,7 +264,7 @@ public class LuceneIndexForPartitionedRegionTest {
LuceneIndexForPartitionedRegion spy = setupSpy(region, index);
verify(spy).createChunkRegion(eq(RegionShortcut.PARTITION), eq(index.createFileRegionName()),
- any(), eq(index.createChunkRegionName()), any());
+ any(), eq(index.createChunkRegionName()), any(), any());
}
@Test
@@ -298,7 +298,7 @@ public class LuceneIndexForPartitionedRegionTest {
partitionAttributes, regionAttributes);
String fileRegionName = index.createFileRegionName();
verify(indexSpy).createRegion(fileRegionName, RegionShortcut.PARTITION, regionPath,
- partitionAttributes, regionAttributes);
+ partitionAttributes, regionAttributes, null);
verify(cache).createVMRegion(eq(fileRegionName), any(), any());
}
@@ -316,9 +316,9 @@ public class LuceneIndexForPartitionedRegionTest {
String chunkRegionName = index.createChunkRegionName();
String fileRegionName = index.createFileRegionName();
indexSpy.createChunkRegion(RegionShortcut.PARTITION, fileRegionName, partitionAttributes,
- chunkRegionName, regionAttributes);
+ chunkRegionName, regionAttributes, null);
verify(indexSpy).createRegion(chunkRegionName, RegionShortcut.PARTITION, fileRegionName,
- partitionAttributes, regionAttributes);
+ partitionAttributes, regionAttributes, null);
verify(cache).createVMRegion(eq(chunkRegionName), any(), any());
}
@@ -335,12 +335,12 @@ public class LuceneIndexForPartitionedRegionTest {
index.setSearchableFields(new String[] {"field"});
LuceneIndexForPartitionedRegion spy = spy(index);
doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
- doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+ doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
doReturn(null).when(spy).createAEQ(any());
spy.initialize();
verify(spy).createChunkRegion(eq(RegionShortcut.PARTITION_PERSISTENT),
- eq(index.createFileRegionName()), any(), eq(index.createChunkRegionName()), any());
+ eq(index.createFileRegionName()), any(), eq(index.createChunkRegionName()), any(), any());
}
@Test
@@ -356,7 +356,7 @@ public class LuceneIndexForPartitionedRegionTest {
index.setSearchableFields(new String[] {"field"});
LuceneIndexForPartitionedRegion spy = spy(index);
doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
- doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+ doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
doReturn(null).when(spy).createAEQ(any());
spy.initialize();
@@ -377,7 +377,7 @@ public class LuceneIndexForPartitionedRegionTest {
index.setSearchableFields(new String[] {"field"});
LuceneIndexForPartitionedRegion spy = spy(index);
doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
- doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+ doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
doReturn(null).when(spy).createAEQ(any());
spy.initialize();
spy.initialize();
http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
index dd52258..7426fa5 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
@@ -27,6 +27,8 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.IntSupplier;
+import org.apache.geode.internal.cache.BucketAdvisor;
+import org.apache.geode.internal.cache.BucketRegion;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -75,7 +77,12 @@ public class IndexRepositoryImplJUnitTest {
String[] indexedFields = new String[] {"s", "i", "l", "d", "f", "s2", "missing"};
mapper = new HeterogeneousLuceneSerializer(indexedFields);
region = Mockito.mock(Region.class);
- userRegion = Mockito.mock(Region.class);
+ userRegion = Mockito.mock(BucketRegion.class);
+ BucketAdvisor bucketAdvisor = Mockito.mock(BucketAdvisor.class);
+ Mockito.when(bucketAdvisor.isPrimary()).thenReturn(true);
+ Mockito.when(((BucketRegion) userRegion).getBucketAdvisor()).thenReturn(bucketAdvisor);
+
+ Mockito.when(((BucketRegion) userRegion).getBucketAdvisor().isPrimary()).thenReturn(true);
stats = Mockito.mock(LuceneIndexStats.class);
Mockito.when(userRegion.isDestroyed()).thenReturn(false);
repo = new IndexRepositoryImpl(region, writer, mapper, stats, userRegion);