You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2019/10/07 21:08:24 UTC
svn commit: r1868100 - in /jackrabbit/oak/trunk:
oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-commons/src/main/java/org/apache/jackrabbit/oak...
Author: reschke
Date: Mon Oct 7 21:08:24 2019
New Revision: 1868100
URL: http://svn.apache.org/viewvc?rev=1868100&view=rev
Log:
OAK-8676: deprecate FileIOUtils.FileLineDifferenceIterator
Added:
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/FileLineDifferenceIterator.java (with props)
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileLineDifferenceIteratorTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1868100&r1=1868099&r2=1868100&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Mon Oct 7 21:08:24 2019
@@ -69,7 +69,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
-import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
+import org.apache.jackrabbit.oak.commons.io.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker;
import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobTracker;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
@@ -103,7 +103,7 @@ public class MarkSweepGarbageCollector i
public static final String DELIM = ",";
- private static final Function<String, String> transformer = new Function<String, String>() {
+ private static final java.util.function.Function<String, String> transformer = new java.util.function.Function<String, String>() {
@Nullable
@Override
public String apply(@Nullable String input) {
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1868100&r1=1868099&r2=1868100&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java Mon Oct 7 21:08:24 2019
@@ -35,9 +35,9 @@ import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.jackrabbit.core.data.DataRecord;
-import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator;
+import org.apache.jackrabbit.oak.commons.io.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import org.apache.jackrabbit.oak.stats.Clock;
import org.jetbrains.annotations.Nullable;
@@ -373,7 +373,7 @@ public class BlobIdTracker implements Cl
/* Lock for operations on the active deletions file */
private ReentrantLock lock;
- private static final Function<String, String> transformer = new Function<String, String>() {
+ private static final java.util.function.Function<String, String> transformer = new java.util.function.Function<String, String>() {
@Nullable
@Override
public String apply(@Nullable String input) {
Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1868100&r1=1868099&r2=1868100&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java Mon Oct 7 21:08:24 2019
@@ -373,8 +373,10 @@ public final class FileIOUtils {
* ensured that both the files are written with
* {@link #writeAsLine(BufferedWriter, String, boolean)} with true to escape line break
* characters.
+ *
+ * @deprecated use {@link org.apache.jackrabbit.oak.commons.io.FileLineDifferenceIterator} instead
*/
- public static class FileLineDifferenceIterator extends AbstractIterator<String> implements Closeable {
+ @Deprecated public static class FileLineDifferenceIterator extends AbstractIterator<String> implements Closeable {
private final PeekingIterator<String> peekMarked;
private final LineIterator marked;
private final LineIterator all;
@@ -397,6 +399,7 @@ public final class FileIOUtils {
public FileLineDifferenceIterator(LineIterator marked, LineIterator available,
@Nullable Function<String, String> transformer) throws IOException {
+ GuavaDeprecation.handleCall("OAK-8676");
this.marked = marked;
this.peekMarked = Iterators.peekingIterator(marked);
this.all = available;
Added: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/FileLineDifferenceIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/FileLineDifferenceIterator.java?rev=1868100&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/FileLineDifferenceIterator.java (added)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/FileLineDifferenceIterator.java Mon Oct 7 21:08:24 2019
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.commons.io;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
+
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.function.Function;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
+import org.jetbrains.annotations.Nullable;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+/**
+ * FileLineDifferenceIterator class which iterates over the difference of 2 files line by line.
+ *
+ * If there is a scope for lines in the files containing line break characters it should be
+ * ensured that both the files are written with
+ * {@link FileIOUtils#writeAsLine(BufferedWriter, String, boolean)} with true to escape line break
+ * characters.
+ */
+public class FileLineDifferenceIterator implements Closeable, Iterator<String> {
+
+ private final Impl delegate;
+
+ public FileLineDifferenceIterator(LineIterator marked, LineIterator available, @Nullable Function<String, String> transformer)
+ throws IOException {
+ this.delegate = new Impl(marked, available, transformer);
+ }
+
+ public FileLineDifferenceIterator(File marked, File available, @Nullable Function<String, String> transformer)
+ throws IOException {
+ this(FileUtils.lineIterator(marked, UTF_8.toString()), FileUtils.lineIterator(available, UTF_8.toString()), transformer);
+ }
+
+ public FileLineDifferenceIterator(LineIterator marked, LineIterator available) throws IOException {
+ this(marked, available, null);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.delegate.hasNext();
+ }
+
+ @Override
+ public String next() {
+ return this.delegate.next();
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.delegate.close();
+ }
+
+ private static class Impl extends AbstractIterator<String> implements Closeable {
+
+ private final PeekingIterator<String> peekMarked;
+ private final LineIterator marked;
+ private final LineIterator all;
+
+ private Function<String, String> transformer = new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ return input;
+ }
+ };
+
+ public Impl(LineIterator marked, LineIterator available, @Nullable Function<String, String> transformer)
+ throws IOException {
+ this.marked = marked;
+ this.peekMarked = Iterators.peekingIterator(marked);
+ this.all = available;
+ if (transformer != null) {
+ this.transformer = transformer;
+ }
+ }
+
+ @Override
+ protected String computeNext() {
+ String diff = computeNextDiff();
+ if (diff == null) {
+ close();
+ return endOfData();
+ }
+ return diff;
+ }
+
+ @Override
+ public void close() {
+ if (marked instanceof Closeable) {
+ closeQuietly(marked);
+ }
+ if (all instanceof Closeable) {
+ closeQuietly(all);
+ }
+ }
+
+ private String computeNextDiff() {
+ if (!all.hasNext()) {
+ return null;
+ }
+
+ // Marked finish the rest of all are part of diff
+ if (!peekMarked.hasNext()) {
+ return all.next();
+ }
+
+ String diff = null;
+ while (all.hasNext() && diff == null) {
+ diff = all.next();
+ while (peekMarked.hasNext()) {
+ String marked = peekMarked.peek();
+ int comparisonResult = transformer.apply(diff).compareTo(transformer.apply((marked)));
+ if (comparisonResult > 0) {
+ // Extra entries in marked. Ignore them and move on
+ peekMarked.next();
+ } else if (comparisonResult == 0) {
+ // Matching entry found in marked move past it. Not a
+ // dif candidate
+ peekMarked.next();
+ diff = null;
+ break;
+ } else {
+ // This entry is not found in marked entries
+ // hence part of diff
+ return diff;
+ }
+ }
+ }
+ return diff;
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/FileLineDifferenceIterator.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileLineDifferenceIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileLineDifferenceIteratorTest.java?rev=1868100&r1=1868099&r2=1868100&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileLineDifferenceIteratorTest.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileLineDifferenceIteratorTest.java Mon Oct 7 21:08:24 2019
@@ -27,15 +27,15 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.TreeSet;
+import java.util.function.Function;
-import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.StandardSystemProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.commons.io.LineIterator;
-import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
+import org.apache.jackrabbit.oak.commons.io.FileLineDifferenceIterator;
import org.jetbrains.annotations.Nullable;
import org.junit.Test;
@@ -136,6 +136,18 @@ public class FileLineDifferenceIteratorT
assertTransformed("", "a:4, b:1", asList("a:4", "b:1"));
}
+ @Test
+ public void testDiffTransformDeprecated() throws IOException {
+ assertTransformedDeprecated("a:x,b:y", "a:1,b:2,c:3,e:4,h", asList("c:3", "e:4", "h"));
+ assertTransformedDeprecated("a,b,d,e", "a,b,c", asList("c"));
+ assertTransformedDeprecated("a:1,b:2,d:3,e:4,f:5", "a:z,b:y,c:x,f:w", asList("c:x"));
+ assertTransformedDeprecated("a,b,d,e,f", "a,b,c,f,h", asList("c", "h"));
+ assertTransformedDeprecated("3:1,7:6", "2:0,3:6,5:3,9:1", asList("2:0", "5:3", "9:1"));
+ assertTransformedDeprecated("", "", Collections.<String> emptyList());
+ assertTransformedDeprecated("", "a, b", asList("a", "b"));
+ assertTransformedDeprecated("", "a:4, b:1", asList("a:4", "b:1"));
+ }
+
private static List<String> getLineBreakStrings() {
return Lists.newArrayList("ab\nc\r", "ab\\z", "a\\\\z\nc",
"/a", "/a/b\nc", "/a/b\rd", "/a/b\r\ne", "/a/c");
@@ -185,6 +197,21 @@ public class FileLineDifferenceIteratorT
@Nullable @Override
public String apply(@Nullable String input) {
if (input != null) {
+ return input.split(":")[0];
+ }
+ return null;
+ }
+ });
+
+ assertThat("marked: " + marked + " all: " + all, ImmutableList.copyOf(itr), is(diff));
+ }
+
+ private static void assertTransformedDeprecated(String marked, String all, List<String> diff) throws IOException {
+ Iterator<String> itr = new org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator(lineItr(marked), lineItr(all),
+ new com.google.common.base.Function<String, String>() {
+ @Nullable @Override
+ public String apply(@Nullable String input) {
+ if (input != null) {
return input.split(":")[0];
}
return null;
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1868100&r1=1868099&r2=1868100&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java Mon Oct 7 21:08:24 2019
@@ -69,7 +69,7 @@ import org.apache.jackrabbit.oak.api.Blo
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
-import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
+import org.apache.jackrabbit.oak.commons.io.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
@@ -361,7 +361,7 @@ public class DataStoreCheckCommand imple
System.out.println("Starting consistency check");
Stopwatch watch = createStarted();
- FileLineDifferenceIterator iter = new FileLineDifferenceIterator(ids, refs, new Function<String, String>() {
+ FileLineDifferenceIterator iter = new FileLineDifferenceIterator(ids, refs, new java.util.function.Function<String, String>() {
@Nullable
@Override
public String apply(@Nullable String input) {
@@ -386,7 +386,7 @@ public class DataStoreCheckCommand imple
// If a delete file is present filter the tracked deleted ids
if (!files.isEmpty()) {
File delFile = files.iterator().next();
- FileLineDifferenceIterator filteringIter = new FileLineDifferenceIterator(delFile, candTemp, new Function<String, String>() {
+ FileLineDifferenceIterator filteringIter = new FileLineDifferenceIterator(delFile, candTemp, new java.util.function.Function<String, String>() {
@Nullable @Override public String apply(@Nullable String input) {
if (input != null) {
return encodeId(decodeId(input.split(DELIM)[0]), dsType);