You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/01/16 13:30:07 UTC
svn commit: r1779027 - in
/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene:
./ directory/ hybrid/ reader/
Author: chetanm
Date: Mon Jan 16 13:30:06 2017
New Revision: 1779027
URL: http://svn.apache.org/viewvc?rev=1779027&view=rev
Log:
OAK-4809 - JMX Stats for NRT Indexing
Modify the existing MBean impl to display NRT related stats and handle directory size calculation properly
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReader.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/LuceneIndexReader.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java?rev=1779027&r1=1779026&r2=1779027&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java Mon Jan 16 13:30:06 2017
@@ -176,6 +176,10 @@ public class IndexNode {
}
}
+ List<LuceneIndexReader> getPrimaryReaders() {
+ return readers;
+ }
+
@CheckForNull
public LuceneIndexWriter getLocalWriter() throws IOException{
return nrtIndex != null ? nrtIndex.getWriter() : null;
@@ -217,7 +221,7 @@ public class IndexNode {
return new MultiReader(readerArr, true);
}
- private List<LuceneIndexReader> getNRTReaders() {
+ List<LuceneIndexReader> getNRTReaders() {
return nrtIndex != null ? nrtIndex.getReaders() : Collections.<LuceneIndexReader>emptyList();
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1779027&r1=1779026&r2=1779027&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java Mon Jan 16 13:30:06 2017
@@ -48,6 +48,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.json.JsopDiff;
import org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo;
import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex.PathStoredFieldVisitor;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.lucene.index.DirectoryReader;
@@ -72,6 +73,7 @@ import static com.google.common.base.Pre
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.INDEX_DEFINITION_NODE;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize;
public class LuceneIndexMBeanImpl extends AnnotatedStandardMBean implements LuceneIndexMBean {
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -95,8 +97,7 @@ public class LuceneIndexMBeanImpl extend
try {
indexNode = indexTracker.acquireIndexNode(path);
if (indexNode != null) {
- IndexStats stats = new IndexStats(path, indexNode.getSearcher().getIndexReader(),
- indexNode.getSuggestDirectory());
+ IndexStats stats = new IndexStats(path, indexNode);
tds.put(stats.toCompositeData());
}
} finally {
@@ -251,7 +252,7 @@ public class LuceneIndexMBeanImpl extend
indexNode = indexTracker.acquireIndexNode(sourcePath);
if (indexNode != null) {
log.info("Dumping Lucene directory content for [{}] to [{}]", sourcePath, destPath);
- Directory source = getDirectory(indexNode.getSearcher().getIndexReader());
+ Directory source = getDirectory(getPrimaryReader(indexNode.getPrimaryReaders()));
checkNotNull(source, "IndexSearcher not backed by DirectoryReader");
Directory dest = FSDirectory.open(new File(destPath));
for (String file : source.listAll()) {
@@ -425,6 +426,9 @@ public class LuceneIndexMBeanImpl extend
"numDocs",
"maxDoc",
"numDeletedDocs",
+ "nrtIndexSize",
+ "nrtIndexSizeStr",
+ "nrtNumDocs"
};
static final String[] FIELD_DESCRIPTIONS = new String[]{
@@ -436,6 +440,9 @@ public class LuceneIndexMBeanImpl extend
"Number of documents in this index.",
"The time and date for when the longest query took place",
"Number of deleted documents",
+ "NRT Index Size in bytes",
+ "NRT Index Size in human readable format",
+ "Number of documents in NRT index"
};
@SuppressWarnings("rawtypes")
@@ -448,6 +455,9 @@ public class LuceneIndexMBeanImpl extend
SimpleType.INTEGER,
SimpleType.INTEGER,
SimpleType.INTEGER,
+ SimpleType.LONG,
+ SimpleType.STRING,
+ SimpleType.INTEGER
};
static final CompositeType TYPE = createCompositeType();
@@ -473,17 +483,22 @@ public class LuceneIndexMBeanImpl extend
private final String indexSizeStr;
private final long suggesterSize;
private final String suggesterSizeStr;
+ private final long nrtIndexSize;
+ private final String nrtIndexSizeStr;
+ private final int numDocsNRT;
- public IndexStats(String path, IndexReader indexReader, @Nullable Directory suggestDirectory)
- throws IOException {
+ public IndexStats(String path, IndexNode indexNode) throws IOException {
this.path = path;
- numDocs = indexReader.numDocs();
- maxDoc = indexReader.maxDoc();
- numDeletedDocs = indexReader.numDeletedDocs();
- indexSize = dirSize(getDirectory(indexReader));
+ numDocs = indexNode.getSearcher().getIndexReader().numDocs();
+ maxDoc = indexNode.getSearcher().getIndexReader().maxDoc();
+ numDeletedDocs = indexNode.getSearcher().getIndexReader().numDeletedDocs();
+ indexSize = getIndexSize(indexNode.getPrimaryReaders());
indexSizeStr = humanReadableByteCount(indexSize);
- suggesterSize = dirSize(suggestDirectory);
+ suggesterSize = dirSize(indexNode.getSuggestDirectory());
suggesterSizeStr = humanReadableByteCount(suggesterSize);
+ nrtIndexSize = getIndexSize(indexNode.getNRTReaders());
+ numDocsNRT = getNumDocs(indexNode.getNRTReaders());
+ nrtIndexSizeStr = humanReadableByteCount(nrtIndexSize);
}
CompositeDataSupport toCompositeData() {
@@ -495,7 +510,10 @@ public class LuceneIndexMBeanImpl extend
suggesterSize,
numDocs,
maxDoc,
- numDeletedDocs
+ numDeletedDocs,
+ nrtIndexSize,
+ nrtIndexSizeStr,
+ numDocsNRT
};
try {
return new CompositeDataSupport(TYPE, FIELD_NAMES, values);
@@ -565,6 +583,18 @@ public class LuceneIndexMBeanImpl extend
}
//~---------------------------------------------------------< Internal >
+ private static IndexReader getPrimaryReader(List<LuceneIndexReader> indexReaders) {
+ return indexReaders.isEmpty() ? null : indexReaders.get(0).getReader();
+ }
+
+ private static long getIndexSize(List<LuceneIndexReader> readers) throws IOException {
+ long totalSize = 0;
+ for (LuceneIndexReader r : readers){
+ totalSize += r.getIndexSize();
+ }
+ return totalSize;
+ }
+
private static Directory getDirectory(IndexReader reader) {
if (reader instanceof DirectoryReader) {
return ((DirectoryReader) reader).directory();
@@ -572,18 +602,12 @@ public class LuceneIndexMBeanImpl extend
return null;
}
- private static long dirSize(Directory directory) throws IOException {
- long totalFileSize = 0L;
- if (directory == null) {
- return -1;
- }
- String[] files = directory.listAll();
- if (files == null) {
- return totalFileSize;
+ private static int getNumDocs(List<LuceneIndexReader> readers) {
+ int numDoc = 0;
+ for (LuceneIndexReader r : readers){
+ numDoc += r.getReader().numDocs();
}
- for (String file : files) {
- totalFileSize += directory.fileLength(file);
- }
- return totalFileSize;
+ return numDoc;
}
+
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java?rev=1779027&r1=1779026&r2=1779027&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java Mon Jan 16 13:30:06 2017
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+import java.io.IOException;
+
import org.apache.lucene.store.Directory;
public class DirectoryUtils {
@@ -38,4 +40,19 @@ public class DirectoryUtils {
}
return -1;
}
+
+ public static long dirSize(Directory directory) throws IOException {
+ long totalFileSize = 0L;
+ if (directory == null) {
+ return -1;
+ }
+ String[] files = directory.listAll();
+ if (files == null) {
+ return totalFileSize;
+ }
+ for (String file : files) {
+ totalFileSize += directory.fileLength(file);
+ }
+ return totalFileSize;
+ }
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java?rev=1779027&r1=1779026&r2=1779027&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java Mon Jan 16 13:30:06 2017
@@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize;
public class NRTIndex implements Closeable {
@@ -85,7 +86,7 @@ public class NRTIndex implements Closeab
@CheckForNull
LuceneIndexReader getPrimaryReader() {
DirectoryReader reader = createReader();
- return reader != null ? new NRTReader(reader) : null;
+ return reader != null ? new NRTReader(reader, directory) : null;
}
public LuceneIndexWriter getWriter() throws IOException {
@@ -111,7 +112,7 @@ public class NRTIndex implements Closeab
}
List<LuceneIndexReader> newReaders = Lists.newArrayListWithCapacity(2);
if (latestReader != null) {
- newReaders.add(new NRTReader(latestReader));
+ newReaders.add(new NRTReader(latestReader, directory));
}
//Old reader should be added later
@@ -220,9 +221,11 @@ public class NRTIndex implements Closeab
private static class NRTReader implements LuceneIndexReader {
private final IndexReader indexReader;
+ private final Directory directory;
- public NRTReader(IndexReader indexReader) {
+ public NRTReader(IndexReader indexReader, Directory directory) {
this.indexReader = checkNotNull(indexReader);
+ this.directory = directory;
}
@Override
@@ -241,6 +244,11 @@ public class NRTIndex implements Closeab
}
@Override
+ public long getIndexSize() throws IOException {
+ return dirSize(directory);
+ }
+
+ @Override
public void close() throws IOException {
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReader.java?rev=1779027&r1=1779026&r2=1779027&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReader.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReader.java Mon Jan 16 13:30:06 2017
@@ -31,6 +31,8 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
import org.apache.lucene.store.Directory;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize;
+
public class DefaultIndexReader implements LuceneIndexReader {
private final Directory directory;
private final Directory suggestDirectory;
@@ -66,6 +68,11 @@ public class DefaultIndexReader implemen
}
@Override
+ public long getIndexSize() throws IOException {
+ return dirSize(directory);
+ }
+
+ @Override
public void close() throws IOException {
try {
reader.close();
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/LuceneIndexReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/LuceneIndexReader.java?rev=1779027&r1=1779026&r2=1779027&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/LuceneIndexReader.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/LuceneIndexReader.java Mon Jan 16 13:30:06 2017
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.reader;
import java.io.Closeable;
+import java.io.IOException;
import javax.annotation.CheckForNull;
@@ -36,4 +37,6 @@ public interface LuceneIndexReader exten
@CheckForNull
Directory getSuggestDirectory();
+
+ long getIndexSize() throws IOException;
}