You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2019/10/07 16:24:20 UTC
svn commit: r1868086 - in /jackrabbit/oak/trunk:
oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/i...
Author: reschke
Date: Mon Oct 7 16:24:20 2019
New Revision: 1868086
URL: http://svn.apache.org/viewvc?rev=1868086&view=rev
Log:
OAK-8666: deprecate FileIOUtils.BurnOnCloseFileIterator
Added:
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java (with props)
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1868086&r1=1868085&r2=1868086&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java Mon Oct 7 16:24:20 2019
@@ -37,6 +37,7 @@ import com.google.common.io.Files;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import org.apache.jackrabbit.oak.stats.Clock;
import org.jetbrains.annotations.Nullable;
@@ -66,7 +67,6 @@ import static org.apache.commons.io.File
import static org.apache.commons.io.FilenameUtils.concat;
import static org.apache.commons.io.FilenameUtils.removeExtension;
import static org.apache.commons.io.IOUtils.closeQuietly;
-import static org.apache.jackrabbit.oak.commons.FileIOUtils.BurnOnCloseFileIterator.wrap;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.append;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.copy;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.sort;
@@ -541,7 +541,7 @@ public class BlobIdTracker implements Cl
try {
// Get a temp file path
String path = createTempFile("temp", null).getAbsolutePath();
- return wrap(lineIterator(getRecords(path)), new File(path));
+ return BurnOnCloseFileIterator.wrap(lineIterator(getRecords(path)), new File(path));
} catch (IOException e) {
LOG.error("Error in retrieving blob records iterator", e);
throw e;
Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1868086&r1=1868085&r2=1868086&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java Mon Oct 7 16:24:20 2019
@@ -456,21 +456,22 @@ public final class FileIOUtils {
* A custom transformer can also be provided to unescape.
*
* @param <T> the type of elements in the iterator
+ * @deprecated use {@link org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator} instead
*/
- public static class BurnOnCloseFileIterator<T> extends AbstractIterator<T> implements Closeable {
+ @Deprecated public static class BurnOnCloseFileIterator<T> extends AbstractIterator<T> implements Closeable {
private final Logger log = LoggerFactory.getLogger(getClass());
private final LineIterator iterator;
private final Function<String, T> transformer;
- private File backingFile;
+ private final File backingFile;
public BurnOnCloseFileIterator(LineIterator iterator, Function<String, T> transformer) {
- this.iterator = iterator;
- this.transformer = transformer;
+ this(iterator, null, transformer);
}
public BurnOnCloseFileIterator(LineIterator iterator, File backingFile,
Function<String, T> transformer) {
+ GuavaDeprecation.handleCall("OAK-8666");
this.iterator = iterator;
this.transformer = transformer;
this.backingFile = backingFile;
Added: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java?rev=1868086&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java (added)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java Mon Oct 7 16:24:20 2019
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.commons.io;
+
+import static org.apache.commons.io.FileUtils.forceDelete;
+import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
+
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.function.Function;
+
+import org.apache.commons.io.LineIterator;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.AbstractIterator;
+
+/**
+ * Implements a {@link java.io.Closeable} wrapper over a {@link LineIterator}.
+ * Also has a transformer to transform the output. If the underlying file is
+ * provided then it deletes the file on {@link #close()}.
+ *
+ * If there is a scope for lines in the file containing line break characters it
+ * should be ensured that the files is written with
+ * {@link FileIOUtils#writeAsLine(BufferedWriter, String, boolean)} with true to escape
+ * line break characters and should be properly unescaped on read. A custom
+ * transformer can also be provided to unescape.
+ *
+ * @param <T>
+ * the type of elements in the iterator
+ */
+public class BurnOnCloseFileIterator<T> implements Closeable, Iterator<T> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BurnOnCloseFileIterator.class);
+
+ private final Impl<T> delegate;
+
+ public BurnOnCloseFileIterator(Iterator<String> iterator, Function<String, T> transformer) {
+ this.delegate = new Impl<T>(iterator, null, transformer);
+ }
+
+ public BurnOnCloseFileIterator(Iterator<String> iterator, File backingFile, Function<String, T> transformer) {
+ this.delegate = new Impl<T>(iterator, backingFile, transformer);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.delegate.hasNext();
+ }
+
+ @Override
+ public T next() {
+ return this.delegate.next();
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.delegate.close();
+ }
+
+ public static BurnOnCloseFileIterator<String> wrap(Iterator<String> iter) {
+ return new BurnOnCloseFileIterator<String>(iter, new Function<String, String>() {
+ public String apply(String s) {
+ return s;
+ }
+ });
+ }
+
+ public static BurnOnCloseFileIterator<String> wrap(Iterator<String> iter, File backingFile) {
+ return new BurnOnCloseFileIterator<String>(iter, backingFile, new Function<String, String>() {
+ public String apply(String s) {
+ return s;
+ }
+ });
+ }
+
+ private static class Impl<T> extends AbstractIterator<T> implements Closeable {
+ private final Iterator<String> iterator;
+ private final Function<String, T> transformer;
+ private final File backingFile;
+
+ public Impl(Iterator<String> iterator, File backingFile, Function<String, T> transformer) {
+ this.iterator = iterator;
+ this.transformer = transformer;
+ this.backingFile = backingFile;
+ }
+
+ @Override
+ protected T computeNext() {
+ if (iterator.hasNext()) {
+ return transformer.apply(iterator.next());
+ }
+
+ try {
+ close();
+ } catch (IOException e) {
+ LOG.warn("Error closing iterator", e);
+ }
+ return endOfData();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (iterator instanceof Closeable) {
+ closeQuietly((Closeable) iterator);
+ }
+ if (backingFile != null && backingFile.exists()) {
+ forceDelete(backingFile);
+ }
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java?rev=1868086&r1=1868085&r2=1868086&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java Mon Oct 7 16:24:20 2019
@@ -338,7 +338,19 @@ public class FileIOUtilsTest {
Set<String> added = newHashSet("a", "z", "e", "b");
File f = assertWrite(added.iterator(), false, added.size());
- BurnOnCloseFileIterator iterator =
+ org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator =
+ org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString()));
+
+ assertEquals(added, Sets.newHashSet(iterator));
+ assertTrue(f.exists());
+ }
+
+ @Test
+ public void deprecatedfileIteratorTest() throws Exception {
+ Set<String> added = newHashSet("a", "z", "e", "b");
+ File f = assertWrite(added.iterator(), false, added.size());
+
+ BurnOnCloseFileIterator<String> iterator =
BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString()));
assertEquals(added, Sets.newHashSet(iterator));
@@ -350,7 +362,19 @@ public class FileIOUtilsTest {
Set<String> added = newHashSet("a", "z", "e", "b");
File f = assertWrite(added.iterator(), false, added.size());
- BurnOnCloseFileIterator iterator =
+ org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator =
+ org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString()), f);
+
+ assertEquals(added, Sets.newHashSet(iterator));
+ assertTrue(!f.exists());
+ }
+
+ @Test
+ public void deprecatedFileIteratorBurnTest() throws Exception {
+ Set<String> added = newHashSet("a", "z", "e", "b");
+ File f = assertWrite(added.iterator(), false, added.size());
+
+ BurnOnCloseFileIterator<String> iterator =
BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString()), f);
assertEquals(added, Sets.newHashSet(iterator));
@@ -362,13 +386,20 @@ public class FileIOUtilsTest {
Set<String> added = newHashSet(getLineBreakStrings());
File f = assertWrite(added.iterator(), true, added.size());
- BurnOnCloseFileIterator iterator =
- new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()), f,
- new Function<String, String>() {
- @Nullable @Override public String apply(@Nullable String input) {
- return unescapeLineBreaks(input);
- }
- });
+ org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator = new org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String>(
+ FileUtils.lineIterator(f, UTF_8.toString()), f, (input) -> unescapeLineBreaks(input));
+
+ assertEquals(added, Sets.newHashSet(iterator));
+ assertTrue(!f.exists());
+ }
+
+ @Test
+ public void deprecatedFileIteratorLineBreakTest() throws IOException {
+ Set<String> added = newHashSet(getLineBreakStrings());
+ File f = assertWrite(added.iterator(), true, added.size());
+
+ BurnOnCloseFileIterator<String> iterator = new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()),
+ f, (input) -> unescapeLineBreaks(input));
assertEquals(added, Sets.newHashSet(iterator));
assertTrue(!f.exists());
@@ -382,16 +413,23 @@ public class FileIOUtilsTest {
}
File f = assertWrite(added.iterator(), true, added.size());
- BurnOnCloseFileIterator iterator =
- new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()),
- f,
- new Function<String, String>() {
- @Nullable
- @Override
- public String apply(@Nullable String input) {
- return unescapeLineBreaks(input);
- }
- });
+ org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator = new org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String>(
+ FileUtils.lineIterator(f, UTF_8.toString()), f, (input) -> unescapeLineBreaks(input));
+
+ assertEquals(added, Sets.newHashSet(iterator));
+ assertTrue(!f.exists());
+ }
+
+ @Test
+ public void deprecatedFileIteratorRandomizedTest() throws Exception {
+ Set<String> added = newHashSet();
+ for (int i = 0; i < 100; i++) {
+ added.add(getRandomTestString());
+ }
+ File f = assertWrite(added.iterator(), true, added.size());
+
+ BurnOnCloseFileIterator<String> iterator = new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()),
+ f, (input) -> unescapeLineBreaks(input));
assertEquals(added, Sets.newHashSet(iterator));
assertTrue(!f.exists());
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java?rev=1868086&r1=1868085&r2=1868086&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java Mon Oct 7 16:24:20 2019
@@ -24,8 +24,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.function.Function;
-import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.io.Closer;
@@ -36,9 +36,9 @@ import org.apache.jackrabbit.oak.api.Blo
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
-import org.apache.jackrabbit.oak.commons.FileIOUtils.BurnOnCloseFileIterator;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator;
import org.apache.jackrabbit.oak.commons.sort.EscapeUtils;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
@@ -336,8 +336,8 @@ public class DataStoreCommand implements
public void log() throws IOException {
File tempFile = new File(outDir, outFile.getName() + "-temp");
FileUtils.moveFile(outFile, tempFile);
- try (BurnOnCloseFileIterator iterator =
- new BurnOnCloseFileIterator(FileUtils.lineIterator(tempFile, UTF_8.toString()), tempFile,
+ try (BurnOnCloseFileIterator<String> iterator =
+ new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(tempFile, UTF_8.toString()), tempFile,
(Function<String, String>) input -> encodeId(input, blobStoreType))) {
FileIOUtils.writeStrings(iterator, outFile, true, log, "Transformed to verbose ids - ");
}