You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2017/08/30 07:06:27 UTC
svn commit: r1806669 - in /jackrabbit/oak/branches/1.6: ./
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
Author: amitj
Date: Wed Aug 30 07:06:27 2017
New Revision: 1806669
URL: http://svn.apache.org/viewvc?rev=1806669&view=rev
Log:
OAK-6596: Blob store consistency check can show bogus errors about missing blobs
Merge r1806668 from trunk
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
jackrabbit/oak/branches/1.6/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 30 07:06:27 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1786122,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788850,1789056,1789534,1790382,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799219,1799389,1799393,1799924,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1805851-1805852
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1786122,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788850,1789056,1789534,1790382,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799219,1799389,1799393,1799924,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1805851-1805852,1806668
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1806669&r1=1806668&r2=1806669&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java (original)
+++ jackrabbit/oak/branches/1.6/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java Wed Aug 30 07:06:27 2017
@@ -95,7 +95,7 @@ public final class FileIOUtils {
*/
public static void sort(File file, Comparator<String> comparator) throws IOException {
File sorted = createTempFile("fleioutilssort", null);
- merge(sortInBatch(file, comparator, true), sorted);
+ merge(sortInBatch(file, comparator, true), sorted, comparator);
move(sorted, file);
}
@@ -113,6 +113,19 @@ public final class FileIOUtils {
}
/**
+ * Merges a list of files after sorting with the given comparator.
+ *
+ * @param files files to merge
+ * @param output merge output file
+ * @throws IOException
+ */
+ public static void merge(List<File> files, File output, Comparator<String> comparator) throws IOException {
+ mergeSortedFiles(
+ files,
+ output, comparator, true);
+ }
+
+ /**
* Copies an input stream to a file.
*
Modified: jackrabbit/oak/branches/1.6/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java?rev=1806669&r1=1806668&r2=1806669&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java (original)
+++ jackrabbit/oak/branches/1.6/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java Wed Aug 30 07:06:27 2017
@@ -40,7 +40,9 @@ import java.util.Set;
import javax.annotation.Nullable;
import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils.BurnOnCloseFileIterator;
import org.junit.Assert;
@@ -65,6 +67,7 @@ import static org.apache.jackrabbit.oak.
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
/**
@@ -160,6 +163,46 @@ public class FileIOUtilsTest {
Collections.sort(list);
assertArrayEquals(Arrays.toString(list.toArray()), list.toArray(), retrieved.toArray());
}
+
+ @Test
+ public void sortLargeFileWithCustomComparatorTest() throws IOException {
+ final int numEntries = 100000; // must be large enough to trigger split/merge functionality of the sort
+ long[] entries = new long[numEntries];
+ Random r = new Random(0);
+ for (int i = 0; i < numEntries; i++) {
+ entries[i] = r.nextLong();
+ }
+
+ Iterator<Long> boxedEntries = Longs.asList(entries).iterator();
+ Iterator<String> hexEntries = Iterators.transform(boxedEntries, new Function<Long, String>() {
+ @Nullable @Override public String apply(@Nullable Long input) {
+ return Long.toHexString(input);
+ }
+ });
+ File f = assertWrite(hexEntries, false, numEntries);
+
+ Comparator<String> prefixComparator = new Comparator<String>() {
+ @Override public int compare(String s1, String s2) {
+ return s1.substring(0, 3).compareTo(s2.substring(0, 3));
+ }
+ };
+
+ sort(f, prefixComparator);
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(new FileInputStream(f), UTF_8));
+ String previous = reader.readLine().substring(0, 3);
+ while (true) {
+ String current = reader.readLine();
+ if (current == null) {
+ break;
+ }
+ current = current.substring(0, 3);
+ assertFalse("Distinct sort didn't filter out duplicates properly.", previous.equals(current));
+ assertTrue("Sort didn't create increasing order", previous.compareTo(current) < 0);
+ previous = current;
+ }
+ closeQuietly(reader);
+ }
@Test
public void testCopy() throws IOException{