You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/12/21 04:45:05 UTC
svn commit: r1818876 - in
/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile:
FlatFileNodeStoreBuilder.java FlatFileStore.java FlatFileStoreUtils.java
Author: chetanm
Date: Thu Dec 21 04:45:05 2017
New Revision: 1818876
URL: http://svn.apache.org/viewvc?rev=1818876&view=rev
Log:
OAK-7102 - Refactor DocumentIndexer logic to enable different sort approaches
Refactor to centralize the reader and writer creation with potential support
for compression (set to false for now)
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java (with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1818876&r1=1818875&r2=1818876&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Thu Dec 21 04:45:05 2017
@@ -19,15 +19,13 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
-import java.io.Writer;
import java.util.Collections;
-import com.google.common.base.Charsets;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
-import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
@@ -71,7 +69,8 @@ public class FlatFileNodeStoreBuilder {
public FlatFileStore build() throws IOException {
log.info("Preferred path elements are {}", Iterables.toString(preferredPathElements));
- FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore), size(preferredPathElements));
+ FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore),
+ size(preferredPathElements), false);
if (entryCount > 0) {
store.setEntryCount(entryCount);
}
@@ -123,7 +122,7 @@ public class FlatFileNodeStoreBuilder {
File file = new File(dir, fileName);
Stopwatch sw = Stopwatch.createStarted();
NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(blobStore);
- try (Writer w = Files.newWriter(file, Charsets.UTF_8)) {
+ try (BufferedWriter w = FlatFileStoreUtils.createWriter(file, false)) {
for (NodeStateEntry e : nodeStates) {
String line = entryWriter.toString(e);
w.append(line).append(LINE_SEPARATOR.value());
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1818876&r1=1818875&r2=1818876&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Thu Dec 21 04:45:05 2017
@@ -21,29 +21,29 @@ package org.apache.jackrabbit.oak.index.
import java.io.Closeable;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.Reader;
import java.util.Iterator;
-import com.google.common.base.Charsets;
import com.google.common.collect.AbstractIterator;
import com.google.common.io.Closer;
-import com.google.common.io.Files;
import org.apache.commons.io.LineIterator;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreUtils.createReader;
+
public class FlatFileStore implements Iterable<NodeStateEntry>, Closeable{
private final Closer closer = Closer.create();
private final File storeFile;
private final NodeStateEntryReader entryReader;
private final int checkChildLimit;
+ private final boolean compressionEnabled;
private long entryCount = -1;
- public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int checkChildLimit) {
+ public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int checkChildLimit, boolean compressionEnabled) {
this.storeFile = storeFile;
this.entryReader = entryReader;
this.checkChildLimit = checkChildLimit;
+ this.compressionEnabled = compressionEnabled;
}
public long getEntryCount() {
@@ -60,7 +60,7 @@ public class FlatFileStore implements It
}
private Iterator<NodeStateEntry> createBaseIterator() {
- LineIterator itr = new LineIterator(createReader());
+ LineIterator itr = new LineIterator(createReader(storeFile, compressionEnabled));
closer.register(itr::close);
return new AbstractIterator<NodeStateEntry>() {
@Override
@@ -84,12 +84,4 @@ public class FlatFileStore implements It
public void close() throws IOException {
closer.close();
}
-
- private Reader createReader() {
- try {
- return Files.newReader(storeFile, Charsets.UTF_8);
- } catch (FileNotFoundException e) {
- throw new RuntimeException("Error opening file " + storeFile, e);
- }
- }
}
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java?rev=1818876&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java Thu Dec 21 04:45:05 2017
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.zip.Deflater;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import static com.google.common.base.Charsets.UTF_8;
+
+class FlatFileStoreUtils {
+
+ public static BufferedReader createReader(File file, boolean compressionEnabled) {
+ try {
+ BufferedReader br;
+ InputStream in = new FileInputStream(file);
+ if (compressionEnabled) {
+ br = new BufferedReader(new InputStreamReader(new GZIPInputStream(in, 2048), UTF_8));
+ } else {
+ br = new BufferedReader(new InputStreamReader(in, UTF_8));
+ }
+ return br;
+ } catch (IOException e) {
+ throw new RuntimeException("Error opening file " + file, e);
+ }
+ }
+
+ public static BufferedWriter createWriter(File file, boolean compressionEnabled) throws IOException {
+ OutputStream out = new FileOutputStream(file);
+ if (compressionEnabled) {
+ out = new GZIPOutputStream(out, 2048) {
+ {
+ def.setLevel(Deflater.BEST_SPEED);
+ }
+ };
+ }
+ return new BufferedWriter(new OutputStreamWriter(out, UTF_8));
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java
------------------------------------------------------------------------------
svn:eol-style = native