You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2016/09/09 23:37:20 UTC
[01/45] accumulo git commit: ACCUMULO-4391 Added synchronization
around the available() call in the BoundedRangeFileInputStream to allow
RFile.Reader deepcopies to be usable on separate threads. Added a test case
which excersizes RFile.Reader deepcopies
Repository: accumulo
Updated Branches:
refs/heads/1.6 726aa1d4c -> 4c040799f
refs/heads/1.7 200be5f29 -> f21c11cc5
refs/heads/1.8 ba0f0d45b -> 7dca4ab29
refs/heads/master 67071206d -> aea7cf16f
ACCUMULO-4391 Added synchronization around the available() call in the BoundedRangeFileInputStream to allow RFile.Reader deepcopies to be usable on separate threads. Added a test case which excersizes RFile.Reader deepcopies across threads.
ACCUMULO-4391: review comments
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5957c223
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5957c223
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5957c223
Branch: refs/heads/1.6
Commit: 5957c2234e9c577ef65c1e0fb44c7c25874059e0
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Wed Jul 27 20:25:46 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 13 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 387 +++++++++++++++++++
2 files changed, 395 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..285040f 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -62,12 +62,15 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
+ final FSDataInputStream inLocal = in;
+ synchronized (inLocal) {
+ int avail = inLocal.available();
+ if (pos + avail > end) {
+ avail = (int) (end - pos);
+ }
- return avail;
+ return avail;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..662bb1b
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ static {
+ Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
+ Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
+ }
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ System.out.println(messages.get(message) + ": " + message);
+ System.out.println(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
[27/45] accumulo git commit: ACCUMULO-4391 removed extra closed check
Posted by ct...@apache.org.
ACCUMULO-4391 removed extra closed check
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3b694418
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3b694418
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3b694418
Branch: refs/heads/1.6
Commit: 3b6944180b4df9d0e56c2296d967b0ec880bd4bd
Parents: 5b08b17
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 21:36:50 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 21:36:50 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +++++++++-----------
1 file changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3b694418/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 60376d5..7fdfc96 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -674,19 +674,17 @@ public final class BCFile {
}
public void finish() throws IOException {
- if (!closed) {
- synchronized (in) {
- if (!closed) {
- try {
- in.close();
- } finally {
- closed = true;
- if (decompressor != null) {
- try {
- compressAlgo.returnDecompressor(decompressor);
- } finally {
- decompressor = null;
- }
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
}
}
}
[30/45] accumulo git commit: Merge branch 'pr-134' into 1.6
Posted by ct...@apache.org.
Merge branch 'pr-134' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9fd94ed4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9fd94ed4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9fd94ed4
Branch: refs/heads/1.7
Commit: 9fd94ed44b342cf4858458f5b923e79c12638844
Parents: 726aa1d 3b69441
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 15:33:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 15:33:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[08/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads.
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6ff35f0f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6ff35f0f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6ff35f0f
Branch: refs/heads/1.7
Commit: 6ff35f0f013497871df22acc842341ea7163a5ab
Parents: 5957c22
Author: Ivan Bella <iv...@bella.name>
Authored: Mon Aug 1 20:30:53 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Mon Aug 1 20:30:53 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +++-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 25 ++++++++++++----
.../bcfile/BoundedRangeFileInputStream.java | 30 +++++++++++---------
3 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..b8cd639 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,7 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private final InputStream in;
+ private InputStream in;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -672,11 +672,24 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ final InputStream inLocal = in;
+ if (inLocal != null) {
+ synchronized (inLocal) {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 285040f..b182b26 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -60,17 +60,15 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
+ private void check() throws IOException {
+ if (in == null) {
+ throw new IOException("Stream closed");
+ }
+ }
+
@Override
public int available() throws IOException {
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- int avail = inLocal.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
- }
+ return (int) (end - pos);
}
@Override
@@ -97,7 +95,9 @@ class BoundedRangeFileInputStream extends InputStream {
return -1;
Integer ret = 0;
final FSDataInputStream inLocal = in;
+ check(); // ensuring inLocal is not null
synchronized (inLocal) {
+ check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
inLocal.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@@ -149,9 +149,13 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ final FSDataInputStream inLocal = in;
+ if (inLocal != null) {
+ // synchronize on the FSDataInputStream to ensure we block closing if in the read method
+ synchronized (inLocal) {
+ // Invalidate the state of the stream.
+ in = null;
+ }
+ }
}
}
[17/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads. Added a test case which forces at least one of the race conditions.
The remaining conditions have been tested
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads. Added a test case which forces at least one of the race conditions. The remaining conditions have been tested on systems that demonstrated these issues.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3c8bc120
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3c8bc120
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3c8bc120
Branch: refs/heads/1.6
Commit: 3c8bc120b1527eb0cba5390da8e57eb947734af8
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:24:16 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 426 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -615,6 +615,7 @@ public final class BCFile {
private Decompressor decompressor;
private final BlockRegion region;
private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,11 +674,23 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
- private FSDataInputStream in;
+ private volatile boolean closed = false;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -62,12 +63,7 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
+ return (int) (end - pos);
}
@Override
@@ -93,15 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -146,9 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..498551f
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[14/45] accumulo git commit: ACCUMJULO-4391 Resolved the remaining
pull request comments.
Posted by ct...@apache.org.
ACCUMJULO-4391 Resolved the remaining pull request comments.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/feb2a51c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/feb2a51c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/feb2a51c
Branch: refs/heads/1.8
Commit: feb2a51cf6b4cbccd181f541b1929bf3fc166b8d
Parents: 9038d8b
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:09:15 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:09:15 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 13 +++++++------
.../rfile/bcfile/BoundedRangeFileInputStream.java | 14 +++++++++-----
.../core/file/rfile/MultiThreadedRFileTest.java | 11 +++--------
.../apache/accumulo/core/file/rfile/RFileTest.java | 7 -------
core/src/test/resources/log4j.properties | 2 ++
5 files changed, 21 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index b8cd639..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,8 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private InputStream in;
+ private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,14 +674,13 @@ public final class BCFile {
}
public void finish() throws IOException {
- final InputStream inLocal = in;
- if (inLocal != null) {
- synchronized (inLocal) {
- if (in != null) {
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
try {
in.close();
} finally {
- in = null;
+ closed = true;
if (decompressor != null) {
try {
compressAlgo.returnDecompressor(decompressor);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 876b585..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
class BoundedRangeFileInputStream extends InputStream {
private volatile boolean closed = false;
- private FSDataInputStream in;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -145,10 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
- synchronized (in) {
- // Invalidate the state of the stream.
- closed = true;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 662bb1b..498551f 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
@@ -61,16 +60,12 @@ import static org.junit.Assert.assertTrue;
public class MultiThreadedRFileTest {
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
private static void checkIndex(Reader reader) throws IOException {
FileSKVIterator indexIter = reader.getIndex();
@@ -269,8 +264,8 @@ public class MultiThreadedRFileTest {
}
for (String message : messages.keySet()) {
- System.out.println(messages.get(message) + ": " + message);
- System.out.println(stackTrace.get(message));
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
}
assertTrue(threadExceptions.isEmpty());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[39/45] accumulo git commit: ACCUMULO-4391 Remove unused field
Posted by ct...@apache.org.
ACCUMULO-4391 Remove unused field
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4c040799
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4c040799
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4c040799
Branch: refs/heads/master
Commit: 4c040799f48c21c5f157fabcab8fde5259b6c7ed
Parents: 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:28 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:28 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4c040799/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 498551f..a0efeed 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -97,7 +96,6 @@ public class MultiThreadedRFileTest {
private Configuration conf = CachedConfiguration.getInstance();
public RFile.Writer writer;
private FSDataOutputStream dos;
- private FSDataInputStream in;
private AccumuloConfiguration accumuloConfiguration;
public Reader reader;
public SortedKeyValueIterator<Key,Value> iter;
[26/45] accumulo git commit: ACCUMULO-4391 removed extra closed check
Posted by ct...@apache.org.
ACCUMULO-4391 removed extra closed check
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3b694418
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3b694418
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3b694418
Branch: refs/heads/1.8
Commit: 3b6944180b4df9d0e56c2296d967b0ec880bd4bd
Parents: 5b08b17
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 21:36:50 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 21:36:50 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +++++++++-----------
1 file changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3b694418/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 60376d5..7fdfc96 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -674,19 +674,17 @@ public final class BCFile {
}
public void finish() throws IOException {
- if (!closed) {
- synchronized (in) {
- if (!closed) {
- try {
- in.close();
- } finally {
- closed = true;
- if (decompressor != null) {
- try {
- compressAlgo.returnDecompressor(decompressor);
- } finally {
- decompressor = null;
- }
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
}
}
}
[40/45] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ct...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f21c11cc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f21c11cc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f21c11cc
Branch: refs/heads/master
Commit: f21c11cc5a9d896f22ab38f2eb56b4f044a0602b
Parents: b0981d7 4c04079
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:57 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:57 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
[28/45] accumulo git commit: ACCUMULO-4391 removed extra closed check
Posted by ct...@apache.org.
ACCUMULO-4391 removed extra closed check
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3b694418
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3b694418
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3b694418
Branch: refs/heads/master
Commit: 3b6944180b4df9d0e56c2296d967b0ec880bd4bd
Parents: 5b08b17
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 21:36:50 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 21:36:50 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +++++++++-----------
1 file changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3b694418/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 60376d5..7fdfc96 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -674,19 +674,17 @@ public final class BCFile {
}
public void finish() throws IOException {
- if (!closed) {
- synchronized (in) {
- if (!closed) {
- try {
- in.close();
- } finally {
- closed = true;
- if (decompressor != null) {
- try {
- compressAlgo.returnDecompressor(decompressor);
- } finally {
- decompressor = null;
- }
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
}
}
}
[42/45] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ct...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f21c11cc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f21c11cc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f21c11cc
Branch: refs/heads/1.8
Commit: f21c11cc5a9d896f22ab38f2eb56b4f044a0602b
Parents: b0981d7 4c04079
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:57 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:57 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
[24/45] accumulo git commit: Merge branch '1.6' of
https://github.com/ivakegg/accumulo into 1.6
Posted by ct...@apache.org.
Merge branch '1.6' of https://github.com/ivakegg/accumulo into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5b08b17b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5b08b17b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5b08b17b
Branch: refs/heads/1.6
Commit: 5b08b17bd5d2d528699153adf8bb541402601b4c
Parents: 3c8bc12 feb2a51
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:32:21 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:32:21 2016 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[29/45] accumulo git commit: Merge branch 'pr-134' into 1.6
Posted by ct...@apache.org.
Merge branch 'pr-134' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9fd94ed4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9fd94ed4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9fd94ed4
Branch: refs/heads/master
Commit: 9fd94ed44b342cf4858458f5b923e79c12638844
Parents: 726aa1d 3b69441
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 15:33:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 15:33:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[35/45] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ct...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b0981d7d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b0981d7d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b0981d7d
Branch: refs/heads/1.8
Commit: b0981d7ddf27caf71d3e53519572880262fc6cc3
Parents: 200be5f 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 16:55:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 16:55:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
[32/45] accumulo git commit: Merge branch 'pr-134' into 1.6
Posted by ct...@apache.org.
Merge branch 'pr-134' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9fd94ed4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9fd94ed4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9fd94ed4
Branch: refs/heads/1.6
Commit: 9fd94ed44b342cf4858458f5b923e79c12638844
Parents: 726aa1d 3b69441
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 15:33:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 15:33:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[33/45] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ct...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b0981d7d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b0981d7d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b0981d7d
Branch: refs/heads/1.7
Commit: b0981d7ddf27caf71d3e53519572880262fc6cc3
Parents: 200be5f 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 16:55:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 16:55:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
[43/45] accumulo git commit: Merge branch '1.7' into 1.8
Posted by ct...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7dca4ab2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7dca4ab2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7dca4ab2
Branch: refs/heads/master
Commit: 7dca4ab2920037386518a36aa1f651e10872fe0b
Parents: ba0f0d4 f21c11c
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:22:41 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:22:41 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../streams/BoundedRangeFileInputStream.java | 35 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 390 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 433 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 77de47e,85a833e..bba8aeb
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -615,9 -612,10 +615,10 @@@ public final class BCFile
private Decompressor decompressor;
private final BlockRegion region;
private final InputStream in;
+ private volatile boolean closed;
- public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
- Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
+ public <InputStreamType extends InputStream & Seekable> RBlockState(Algorithm compressionAlgo, InputStreamType fsin, BlockRegion region,
+ Configuration conf, CryptoModule cryptoModule, Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
this.compressAlgo = compressionAlgo;
this.region = region;
this.decompressor = compressionAlgo.getDecompressor();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
index 1c01843,0000000..a033ad4
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
@@@ -1,152 -1,0 +1,157 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.streams;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.fs.Seekable;
+
+/**
+ * BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop FSDataInputStream as a regular input stream. One can create multiple
+ * BoundedRangeFileInputStream on top of the same FSDataInputStream and they would not interfere with each other.
+ */
+public class BoundedRangeFileInputStream extends InputStream {
- private InputStream in;
++
++ private volatile boolean closed = false;
++ private final InputStream in;
+ private long pos;
+ private long end;
+ private long mark;
+ private final byte[] oneByte = new byte[1];
+
+ /**
+ * Constructor
+ *
+ * @param in
+ * The FSDataInputStream we connect to.
+ * @param offset
+ * Beginning offset of the region.
+ * @param length
+ * Length of the region.
+ *
+ * The actual length of the region may be smaller if (off_begin + length) goes beyond the end of FS input stream.
+ */
+ public <StreamType extends InputStream & Seekable> BoundedRangeFileInputStream(StreamType in, long offset, long length) {
+ if (offset < 0 || length < 0) {
+ throw new IndexOutOfBoundsException("Invalid offset/length: " + offset + "/" + length);
+ }
+
+ this.in = in;
+ this.pos = offset;
+ this.end = offset + length;
+ this.mark = -1;
+ }
+
+ @Override
+ public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
++ return (int) (end - pos);
+ }
+
+ @Override
+ public int read() throws IOException {
+ int ret = read(oneByte);
+ if (ret == 1)
+ return oneByte[0] & 0xff;
+ return -1;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, int len) throws IOException {
+ if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ final int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
+ if (n == 0)
+ return -1;
+ Integer ret = 0;
- final InputStream inLocal = in;
- synchronized (inLocal) {
- ((Seekable) inLocal).seek(pos);
++ synchronized (in) {
++ // ensuring we are not closed which would be followed by someone else reusing the decompressor
++ if (closed) {
++ throw new IOException("Stream closed");
++ }
++ ((Seekable) in).seek(pos);
+ try {
+ ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws IOException {
+ int ret = 0;
- ret = inLocal.read(b, off, n);
++ ret = in.read(b, off, n);
+ return ret;
+ }
+ });
+ } catch (PrivilegedActionException e) {
+ throw (IOException) e.getException();
+ }
+ }
+ if (ret < 0) {
+ end = pos;
+ return -1;
+ }
+ pos += ret;
+ return ret;
+ }
+
+ @Override
+ /*
+ * We may skip beyond the end of the file.
+ */
+ public long skip(long n) throws IOException {
+ long len = Math.min(n, end - pos);
+ pos += len;
+ return len;
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ mark = pos;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ if (mark < 0)
+ throw new IOException("Resetting to invalid mark");
+ pos = mark;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ @Override
+ public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
++ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
++ // Once this close completes, the underlying decompression stream may be returned to
++ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
++ if (!closed) {
++ synchronized (in) {
++ // Invalidate the state of the stream.
++ closed = true;
++ }
++ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 0000000,a0efeed..16f2349
mode 000000,100644..100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@@ -1,0 -1,380 +1,390 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.accumulo.core.file.rfile;
+
+ import java.io.File;
+ import java.io.IOException;
+ import java.io.PrintWriter;
+ import java.io.StringWriter;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Random;
+ import java.util.concurrent.LinkedBlockingQueue;
+ import java.util.concurrent.ThreadPoolExecutor;
+ import java.util.concurrent.TimeUnit;
++
++import org.apache.accumulo.core.client.sample.Sampler;
+ import org.apache.accumulo.core.conf.AccumuloConfiguration;
+ import org.apache.accumulo.core.data.ArrayByteSequence;
+ import org.apache.accumulo.core.data.ByteSequence;
+ import org.apache.accumulo.core.data.Key;
+ import org.apache.accumulo.core.data.Range;
+ import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.file.FileSKVIterator;
+ import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+ import org.apache.accumulo.core.file.rfile.RFile.Reader;
++import org.apache.accumulo.core.file.streams.PositionedOutputs;
+ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+ import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
++import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
++import org.apache.accumulo.core.sample.impl.SamplerFactory;
+ import org.apache.accumulo.core.util.CachedConfiguration;
+ import org.apache.accumulo.core.util.NamingThreadFactory;
-import org.apache.commons.lang3.mutable.MutableInt;
++import org.apache.commons.lang.mutable.MutableInt;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.FSDataOutputStream;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.log4j.Logger;
+ import org.junit.Rule;
+ import org.junit.Test;
+ import org.junit.rules.TemporaryFolder;
+
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertTrue;
+
+ public class MultiThreadedRFileTest {
+
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
- private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
++ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
- CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
- writer = new RFile.Writer(_cbw, 1000, 1000);
++ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(PositionedOutputs.wrap(dos), "gz", conf, accumuloConfiguration);
++ SamplerConfigurationImpl samplerConfig = SamplerConfigurationImpl.newSamplerConfig(accumuloConfiguration);
++ Sampler sampler = null;
++ if (samplerConfig != null) {
++ sampler = SamplerFactory.newSampler(samplerConfig, accumuloConfiguration);
++ }
++ writer = new RFile.Writer(_cbw, 1000, 1000, samplerConfig, sampler);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
- Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
- Map<String,String> stackTrace = new HashMap<String,String>();
++ Map<String,MutableInt> messages = new HashMap<>();
++ Map<String,String> stackTrace = new HashMap<>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 069077c,15729c5..3833564
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@@ -77,9 -67,6 +77,7 @@@ import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
- import org.apache.log4j.Level;
- import org.apache.log4j.Logger;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
[06/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads.
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6ff35f0f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6ff35f0f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6ff35f0f
Branch: refs/heads/1.6
Commit: 6ff35f0f013497871df22acc842341ea7163a5ab
Parents: 5957c22
Author: Ivan Bella <iv...@bella.name>
Authored: Mon Aug 1 20:30:53 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Mon Aug 1 20:30:53 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +++-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 25 ++++++++++++----
.../bcfile/BoundedRangeFileInputStream.java | 30 +++++++++++---------
3 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..b8cd639 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,7 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private final InputStream in;
+ private InputStream in;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -672,11 +672,24 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ final InputStream inLocal = in;
+ if (inLocal != null) {
+ synchronized (inLocal) {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 285040f..b182b26 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -60,17 +60,15 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
+ private void check() throws IOException {
+ if (in == null) {
+ throw new IOException("Stream closed");
+ }
+ }
+
@Override
public int available() throws IOException {
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- int avail = inLocal.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
- }
+ return (int) (end - pos);
}
@Override
@@ -97,7 +95,9 @@ class BoundedRangeFileInputStream extends InputStream {
return -1;
Integer ret = 0;
final FSDataInputStream inLocal = in;
+ check(); // ensuring inLocal is not null
synchronized (inLocal) {
+ check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
inLocal.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@@ -149,9 +149,13 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ final FSDataInputStream inLocal = in;
+ if (inLocal != null) {
+ // synchronize on the FSDataInputStream to ensure we block closing if in the read method
+ synchronized (inLocal) {
+ // Invalidate the state of the stream.
+ in = null;
+ }
+ }
}
}
[37/45] accumulo git commit: ACCUMULO-4391 Remove unused field
Posted by ct...@apache.org.
ACCUMULO-4391 Remove unused field
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4c040799
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4c040799
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4c040799
Branch: refs/heads/1.7
Commit: 4c040799f48c21c5f157fabcab8fde5259b6c7ed
Parents: 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:28 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:28 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4c040799/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 498551f..a0efeed 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -97,7 +96,6 @@ public class MultiThreadedRFileTest {
private Configuration conf = CachedConfiguration.getInstance();
public RFile.Writer writer;
private FSDataOutputStream dos;
- private FSDataInputStream in;
private AccumuloConfiguration accumuloConfiguration;
public Reader reader;
public SortedKeyValueIterator<Key,Value> iter;
[07/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads.
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6ff35f0f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6ff35f0f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6ff35f0f
Branch: refs/heads/master
Commit: 6ff35f0f013497871df22acc842341ea7163a5ab
Parents: 5957c22
Author: Ivan Bella <iv...@bella.name>
Authored: Mon Aug 1 20:30:53 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Mon Aug 1 20:30:53 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +++-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 25 ++++++++++++----
.../bcfile/BoundedRangeFileInputStream.java | 30 +++++++++++---------
3 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..b8cd639 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,7 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private final InputStream in;
+ private InputStream in;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -672,11 +672,24 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ final InputStream inLocal = in;
+ if (inLocal != null) {
+ synchronized (inLocal) {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 285040f..b182b26 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -60,17 +60,15 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
+ private void check() throws IOException {
+ if (in == null) {
+ throw new IOException("Stream closed");
+ }
+ }
+
@Override
public int available() throws IOException {
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- int avail = inLocal.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
- }
+ return (int) (end - pos);
}
@Override
@@ -97,7 +95,9 @@ class BoundedRangeFileInputStream extends InputStream {
return -1;
Integer ret = 0;
final FSDataInputStream inLocal = in;
+ check(); // ensuring inLocal is not null
synchronized (inLocal) {
+ check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
inLocal.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@@ -149,9 +149,13 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ final FSDataInputStream inLocal = in;
+ if (inLocal != null) {
+ // synchronize on the FSDataInputStream to ensure we block closing if in the read method
+ synchronized (inLocal) {
+ // Invalidate the state of the stream.
+ in = null;
+ }
+ }
}
}
[19/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads. Added a test case which forces at least one of the race conditions.
The remaining conditions have been tested
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads. Added a test case which forces at least one of the race conditions. The remaining conditions have been tested on systems that demonstrated these issues.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3c8bc120
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3c8bc120
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3c8bc120
Branch: refs/heads/master
Commit: 3c8bc120b1527eb0cba5390da8e57eb947734af8
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:24:16 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 426 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -615,6 +615,7 @@ public final class BCFile {
private Decompressor decompressor;
private final BlockRegion region;
private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,11 +674,23 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
- private FSDataInputStream in;
+ private volatile boolean closed = false;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -62,12 +63,7 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
+ return (int) (end - pos);
}
@Override
@@ -93,15 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -146,9 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..498551f
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[10/45] accumulo git commit: ACCUMULO-4391 Use a closed flag
Posted by ct...@apache.org.
ACCUMULO-4391 Use a closed flag
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9038d8b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9038d8b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9038d8b3
Branch: refs/heads/master
Commit: 9038d8b3a36e7189fbdedfbbb950c7b1f43b2b7d
Parents: 6ff35f0
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Aug 4 00:04:26 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Aug 4 00:04:26 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 31 ++++++++------------
1 file changed, 12 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9038d8b3/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index b182b26..876b585 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
+ private volatile boolean closed = false;
private FSDataInputStream in;
private long pos;
private long end;
@@ -60,12 +61,6 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
- private void check() throws IOException {
- if (in == null) {
- throw new IOException("Stream closed");
- }
- }
-
@Override
public int available() throws IOException {
return (int) (end - pos);
@@ -94,17 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- check(); // ensuring inLocal is not null
- synchronized (inLocal) {
- check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -149,13 +145,10 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- final FSDataInputStream inLocal = in;
- if (inLocal != null) {
- // synchronize on the FSDataInputStream to ensure we block closing if in the read method
- synchronized (inLocal) {
- // Invalidate the state of the stream.
- in = null;
- }
+ // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
}
}
}
[13/45] accumulo git commit: ACCUMJULO-4391 Resolved the remaining
pull request comments.
Posted by ct...@apache.org.
ACCUMJULO-4391 Resolved the remaining pull request comments.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/feb2a51c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/feb2a51c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/feb2a51c
Branch: refs/heads/1.6
Commit: feb2a51cf6b4cbccd181f541b1929bf3fc166b8d
Parents: 9038d8b
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:09:15 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:09:15 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 13 +++++++------
.../rfile/bcfile/BoundedRangeFileInputStream.java | 14 +++++++++-----
.../core/file/rfile/MultiThreadedRFileTest.java | 11 +++--------
.../apache/accumulo/core/file/rfile/RFileTest.java | 7 -------
core/src/test/resources/log4j.properties | 2 ++
5 files changed, 21 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index b8cd639..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,8 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private InputStream in;
+ private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,14 +674,13 @@ public final class BCFile {
}
public void finish() throws IOException {
- final InputStream inLocal = in;
- if (inLocal != null) {
- synchronized (inLocal) {
- if (in != null) {
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
try {
in.close();
} finally {
- in = null;
+ closed = true;
if (decompressor != null) {
try {
compressAlgo.returnDecompressor(decompressor);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 876b585..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
class BoundedRangeFileInputStream extends InputStream {
private volatile boolean closed = false;
- private FSDataInputStream in;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -145,10 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
- synchronized (in) {
- // Invalidate the state of the stream.
- closed = true;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 662bb1b..498551f 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
@@ -61,16 +60,12 @@ import static org.junit.Assert.assertTrue;
public class MultiThreadedRFileTest {
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
private static void checkIndex(Reader reader) throws IOException {
FileSKVIterator indexIter = reader.getIndex();
@@ -269,8 +264,8 @@ public class MultiThreadedRFileTest {
}
for (String message : messages.keySet()) {
- System.out.println(messages.get(message) + ": " + message);
- System.out.println(stackTrace.get(message));
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
}
assertTrue(threadExceptions.isEmpty());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[05/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads.
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6ff35f0f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6ff35f0f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6ff35f0f
Branch: refs/heads/1.8
Commit: 6ff35f0f013497871df22acc842341ea7163a5ab
Parents: 5957c22
Author: Ivan Bella <iv...@bella.name>
Authored: Mon Aug 1 20:30:53 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Mon Aug 1 20:30:53 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +++-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 25 ++++++++++++----
.../bcfile/BoundedRangeFileInputStream.java | 30 +++++++++++---------
3 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..b8cd639 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,7 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private final InputStream in;
+ private InputStream in;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -672,11 +672,24 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ final InputStream inLocal = in;
+ if (inLocal != null) {
+ synchronized (inLocal) {
+ if (in != null) {
+ try {
+ in.close();
+ } finally {
+ in = null;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6ff35f0f/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 285040f..b182b26 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -60,17 +60,15 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
+ private void check() throws IOException {
+ if (in == null) {
+ throw new IOException("Stream closed");
+ }
+ }
+
@Override
public int available() throws IOException {
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- int avail = inLocal.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
- }
+ return (int) (end - pos);
}
@Override
@@ -97,7 +95,9 @@ class BoundedRangeFileInputStream extends InputStream {
return -1;
Integer ret = 0;
final FSDataInputStream inLocal = in;
+ check(); // ensuring inLocal is not null
synchronized (inLocal) {
+ check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
inLocal.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@@ -149,9 +149,13 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ final FSDataInputStream inLocal = in;
+ if (inLocal != null) {
+ // synchronize on the FSDataInputStream to ensure we block closing if in the read method
+ synchronized (inLocal) {
+ // Invalidate the state of the stream.
+ in = null;
+ }
+ }
}
}
[09/45] accumulo git commit: ACCUMULO-4391 Use a closed flag
Posted by ct...@apache.org.
ACCUMULO-4391 Use a closed flag
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9038d8b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9038d8b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9038d8b3
Branch: refs/heads/1.8
Commit: 9038d8b3a36e7189fbdedfbbb950c7b1f43b2b7d
Parents: 6ff35f0
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Aug 4 00:04:26 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Aug 4 00:04:26 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 31 ++++++++------------
1 file changed, 12 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9038d8b3/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index b182b26..876b585 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
+ private volatile boolean closed = false;
private FSDataInputStream in;
private long pos;
private long end;
@@ -60,12 +61,6 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
- private void check() throws IOException {
- if (in == null) {
- throw new IOException("Stream closed");
- }
- }
-
@Override
public int available() throws IOException {
return (int) (end - pos);
@@ -94,17 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- check(); // ensuring inLocal is not null
- synchronized (inLocal) {
- check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -149,13 +145,10 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- final FSDataInputStream inLocal = in;
- if (inLocal != null) {
- // synchronize on the FSDataInputStream to ensure we block closing if in the read method
- synchronized (inLocal) {
- // Invalidate the state of the stream.
- in = null;
- }
+ // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
}
}
}
[15/45] accumulo git commit: ACCUMJULO-4391 Resolved the remaining
pull request comments.
Posted by ct...@apache.org.
ACCUMJULO-4391 Resolved the remaining pull request comments.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/feb2a51c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/feb2a51c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/feb2a51c
Branch: refs/heads/master
Commit: feb2a51cf6b4cbccd181f541b1929bf3fc166b8d
Parents: 9038d8b
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:09:15 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:09:15 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 13 +++++++------
.../rfile/bcfile/BoundedRangeFileInputStream.java | 14 +++++++++-----
.../core/file/rfile/MultiThreadedRFileTest.java | 11 +++--------
.../apache/accumulo/core/file/rfile/RFileTest.java | 7 -------
core/src/test/resources/log4j.properties | 2 ++
5 files changed, 21 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index b8cd639..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,8 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private InputStream in;
+ private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,14 +674,13 @@ public final class BCFile {
}
public void finish() throws IOException {
- final InputStream inLocal = in;
- if (inLocal != null) {
- synchronized (inLocal) {
- if (in != null) {
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
try {
in.close();
} finally {
- in = null;
+ closed = true;
if (decompressor != null) {
try {
compressAlgo.returnDecompressor(decompressor);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 876b585..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
class BoundedRangeFileInputStream extends InputStream {
private volatile boolean closed = false;
- private FSDataInputStream in;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -145,10 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
- synchronized (in) {
- // Invalidate the state of the stream.
- closed = true;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 662bb1b..498551f 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
@@ -61,16 +60,12 @@ import static org.junit.Assert.assertTrue;
public class MultiThreadedRFileTest {
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
private static void checkIndex(Reader reader) throws IOException {
FileSKVIterator indexIter = reader.getIndex();
@@ -269,8 +264,8 @@ public class MultiThreadedRFileTest {
}
for (String message : messages.keySet()) {
- System.out.println(messages.get(message) + ": " + message);
- System.out.println(stackTrace.get(message));
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
}
assertTrue(threadExceptions.isEmpty());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[16/45] accumulo git commit: ACCUMJULO-4391 Resolved the remaining
pull request comments.
Posted by ct...@apache.org.
ACCUMJULO-4391 Resolved the remaining pull request comments.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/feb2a51c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/feb2a51c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/feb2a51c
Branch: refs/heads/1.7
Commit: feb2a51cf6b4cbccd181f541b1929bf3fc166b8d
Parents: 9038d8b
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:09:15 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:09:15 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 13 +++++++------
.../rfile/bcfile/BoundedRangeFileInputStream.java | 14 +++++++++-----
.../core/file/rfile/MultiThreadedRFileTest.java | 11 +++--------
.../apache/accumulo/core/file/rfile/RFileTest.java | 7 -------
core/src/test/resources/log4j.properties | 2 ++
5 files changed, 21 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index b8cd639..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -614,7 +614,8 @@ public final class BCFile {
private final Algorithm compressAlgo;
private Decompressor decompressor;
private final BlockRegion region;
- private InputStream in;
+ private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,14 +674,13 @@ public final class BCFile {
}
public void finish() throws IOException {
- final InputStream inLocal = in;
- if (inLocal != null) {
- synchronized (inLocal) {
- if (in != null) {
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
try {
in.close();
} finally {
- in = null;
+ closed = true;
if (decompressor != null) {
try {
compressAlgo.returnDecompressor(decompressor);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index 876b585..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
class BoundedRangeFileInputStream extends InputStream {
private volatile boolean closed = false;
- private FSDataInputStream in;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -145,10 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
- synchronized (in) {
- // Invalidate the state of the stream.
- closed = true;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 662bb1b..498551f 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
@@ -61,16 +60,12 @@ import static org.junit.Assert.assertTrue;
public class MultiThreadedRFileTest {
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
private static void checkIndex(Reader reader) throws IOException {
FileSKVIterator indexIter = reader.getIndex();
@@ -269,8 +264,8 @@ public class MultiThreadedRFileTest {
}
for (String message : messages.keySet()) {
- System.out.println(messages.get(message) + ": " + message);
- System.out.println(stackTrace.get(message));
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
}
assertTrue(threadExceptions.isEmpty());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/feb2a51c/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[21/45] accumulo git commit: Merge branch '1.6' of
https://github.com/ivakegg/accumulo into 1.6
Posted by ct...@apache.org.
Merge branch '1.6' of https://github.com/ivakegg/accumulo into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5b08b17b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5b08b17b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5b08b17b
Branch: refs/heads/1.7
Commit: 5b08b17bd5d2d528699153adf8bb541402601b4c
Parents: 3c8bc12 feb2a51
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:32:21 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:32:21 2016 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[03/45] accumulo git commit: ACCUMULO-4391 Added synchronization
around the available() call in the BoundedRangeFileInputStream to allow
RFile.Reader deepcopies to be usable on separate threads. Added a test case
which excersizes RFile.Reader deepcopies
Posted by ct...@apache.org.
ACCUMULO-4391 Added synchronization around the available() call in the BoundedRangeFileInputStream to allow RFile.Reader deepcopies to be usable on separate threads. Added a test case which excersizes RFile.Reader deepcopies across threads.
ACCUMULO-4391: review comments
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5957c223
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5957c223
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5957c223
Branch: refs/heads/master
Commit: 5957c2234e9c577ef65c1e0fb44c7c25874059e0
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Wed Jul 27 20:25:46 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 13 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 387 +++++++++++++++++++
2 files changed, 395 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..285040f 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -62,12 +62,15 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
+ final FSDataInputStream inLocal = in;
+ synchronized (inLocal) {
+ int avail = inLocal.available();
+ if (pos + avail > end) {
+ avail = (int) (end - pos);
+ }
- return avail;
+ return avail;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..662bb1b
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ static {
+ Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
+ Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
+ }
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ System.out.println(messages.get(message) + ": " + message);
+ System.out.println(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
[44/45] accumulo git commit: Merge branch '1.7' into 1.8
Posted by ct...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7dca4ab2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7dca4ab2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7dca4ab2
Branch: refs/heads/1.8
Commit: 7dca4ab2920037386518a36aa1f651e10872fe0b
Parents: ba0f0d4 f21c11c
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:22:41 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:22:41 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../streams/BoundedRangeFileInputStream.java | 35 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 390 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 433 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 77de47e,85a833e..bba8aeb
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@@ -615,9 -612,10 +615,10 @@@ public final class BCFile
private Decompressor decompressor;
private final BlockRegion region;
private final InputStream in;
+ private volatile boolean closed;
- public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
- Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
+ public <InputStreamType extends InputStream & Seekable> RBlockState(Algorithm compressionAlgo, InputStreamType fsin, BlockRegion region,
+ Configuration conf, CryptoModule cryptoModule, Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
this.compressAlgo = compressionAlgo;
this.region = region;
this.decompressor = compressionAlgo.getDecompressor();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
index 1c01843,0000000..a033ad4
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/streams/BoundedRangeFileInputStream.java
@@@ -1,152 -1,0 +1,157 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.accumulo.core.file.streams;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.fs.Seekable;
+
+/**
+ * BoundedRangeFIleInputStream abstracts a contiguous region of a Hadoop FSDataInputStream as a regular input stream. One can create multiple
+ * BoundedRangeFileInputStream on top of the same FSDataInputStream and they would not interfere with each other.
+ */
+public class BoundedRangeFileInputStream extends InputStream {
- private InputStream in;
++
++ private volatile boolean closed = false;
++ private final InputStream in;
+ private long pos;
+ private long end;
+ private long mark;
+ private final byte[] oneByte = new byte[1];
+
+ /**
+ * Constructor
+ *
+ * @param in
+ * The FSDataInputStream we connect to.
+ * @param offset
+ * Beginning offset of the region.
+ * @param length
+ * Length of the region.
+ *
+ * The actual length of the region may be smaller if (off_begin + length) goes beyond the end of FS input stream.
+ */
+ public <StreamType extends InputStream & Seekable> BoundedRangeFileInputStream(StreamType in, long offset, long length) {
+ if (offset < 0 || length < 0) {
+ throw new IndexOutOfBoundsException("Invalid offset/length: " + offset + "/" + length);
+ }
+
+ this.in = in;
+ this.pos = offset;
+ this.end = offset + length;
+ this.mark = -1;
+ }
+
+ @Override
+ public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
++ return (int) (end - pos);
+ }
+
+ @Override
+ public int read() throws IOException {
+ int ret = read(oneByte);
+ if (ret == 1)
+ return oneByte[0] & 0xff;
+ return -1;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(final byte[] b, final int off, int len) throws IOException {
+ if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ final int n = (int) Math.min(Integer.MAX_VALUE, Math.min(len, (end - pos)));
+ if (n == 0)
+ return -1;
+ Integer ret = 0;
- final InputStream inLocal = in;
- synchronized (inLocal) {
- ((Seekable) inLocal).seek(pos);
++ synchronized (in) {
++ // ensuring we are not closed which would be followed by someone else reusing the decompressor
++ if (closed) {
++ throw new IOException("Stream closed");
++ }
++ ((Seekable) in).seek(pos);
+ try {
+ ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws IOException {
+ int ret = 0;
- ret = inLocal.read(b, off, n);
++ ret = in.read(b, off, n);
+ return ret;
+ }
+ });
+ } catch (PrivilegedActionException e) {
+ throw (IOException) e.getException();
+ }
+ }
+ if (ret < 0) {
+ end = pos;
+ return -1;
+ }
+ pos += ret;
+ return ret;
+ }
+
+ @Override
+ /*
+ * We may skip beyond the end of the file.
+ */
+ public long skip(long n) throws IOException {
+ long len = Math.min(n, end - pos);
+ pos += len;
+ return len;
+ }
+
+ @Override
+ public void mark(int readlimit) {
+ mark = pos;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ if (mark < 0)
+ throw new IOException("Resetting to invalid mark");
+ pos = mark;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ @Override
+ public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
++ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
++ // Once this close completes, the underlying decompression stream may be returned to
++ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
++ if (!closed) {
++ synchronized (in) {
++ // Invalidate the state of the stream.
++ closed = true;
++ }
++ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 0000000,a0efeed..16f2349
mode 000000,100644..100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@@ -1,0 -1,380 +1,390 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.accumulo.core.file.rfile;
+
+ import java.io.File;
+ import java.io.IOException;
+ import java.io.PrintWriter;
+ import java.io.StringWriter;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Random;
+ import java.util.concurrent.LinkedBlockingQueue;
+ import java.util.concurrent.ThreadPoolExecutor;
+ import java.util.concurrent.TimeUnit;
++
++import org.apache.accumulo.core.client.sample.Sampler;
+ import org.apache.accumulo.core.conf.AccumuloConfiguration;
+ import org.apache.accumulo.core.data.ArrayByteSequence;
+ import org.apache.accumulo.core.data.ByteSequence;
+ import org.apache.accumulo.core.data.Key;
+ import org.apache.accumulo.core.data.Range;
+ import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.file.FileSKVIterator;
+ import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+ import org.apache.accumulo.core.file.rfile.RFile.Reader;
++import org.apache.accumulo.core.file.streams.PositionedOutputs;
+ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+ import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
++import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
++import org.apache.accumulo.core.sample.impl.SamplerFactory;
+ import org.apache.accumulo.core.util.CachedConfiguration;
+ import org.apache.accumulo.core.util.NamingThreadFactory;
-import org.apache.commons.lang3.mutable.MutableInt;
++import org.apache.commons.lang.mutable.MutableInt;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.FSDataOutputStream;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.log4j.Logger;
+ import org.junit.Rule;
+ import org.junit.Test;
+ import org.junit.rules.TemporaryFolder;
+
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertTrue;
+
+ public class MultiThreadedRFileTest {
+
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
- private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
++ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
- CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
- writer = new RFile.Writer(_cbw, 1000, 1000);
++ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(PositionedOutputs.wrap(dos), "gz", conf, accumuloConfiguration);
++ SamplerConfigurationImpl samplerConfig = SamplerConfigurationImpl.newSamplerConfig(accumuloConfiguration);
++ Sampler sampler = null;
++ if (samplerConfig != null) {
++ sampler = SamplerFactory.newSampler(samplerConfig, accumuloConfiguration);
++ }
++ writer = new RFile.Writer(_cbw, 1000, 1000, samplerConfig, sampler);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
- Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
- Map<String,String> stackTrace = new HashMap<String,String>();
++ Map<String,MutableInt> messages = new HashMap<>();
++ Map<String,String> stackTrace = new HashMap<>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7dca4ab2/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 069077c,15729c5..3833564
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@@ -77,9 -67,6 +77,7 @@@ import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
- import org.apache.log4j.Level;
- import org.apache.log4j.Logger;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
[04/45] accumulo git commit: ACCUMULO-4391 Added synchronization
around the available() call in the BoundedRangeFileInputStream to allow
RFile.Reader deepcopies to be usable on separate threads. Added a test case
which excersizes RFile.Reader deepcopies
Posted by ct...@apache.org.
ACCUMULO-4391 Added synchronization around the available() call in the BoundedRangeFileInputStream to allow RFile.Reader deepcopies to be usable on separate threads. Added a test case which excersizes RFile.Reader deepcopies across threads.
ACCUMULO-4391: review comments
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5957c223
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5957c223
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5957c223
Branch: refs/heads/1.8
Commit: 5957c2234e9c577ef65c1e0fb44c7c25874059e0
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Wed Jul 27 20:25:46 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 13 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 387 +++++++++++++++++++
2 files changed, 395 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..285040f 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -62,12 +62,15 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
+ final FSDataInputStream inLocal = in;
+ synchronized (inLocal) {
+ int avail = inLocal.available();
+ if (pos + avail > end) {
+ avail = (int) (end - pos);
+ }
- return avail;
+ return avail;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..662bb1b
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ static {
+ Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
+ Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
+ }
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ System.out.println(messages.get(message) + ": " + message);
+ System.out.println(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
[36/45] accumulo git commit: ACCUMULO-4391 Remove unused field
Posted by ct...@apache.org.
ACCUMULO-4391 Remove unused field
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4c040799
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4c040799
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4c040799
Branch: refs/heads/1.6
Commit: 4c040799f48c21c5f157fabcab8fde5259b6c7ed
Parents: 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:28 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:28 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4c040799/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 498551f..a0efeed 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -97,7 +96,6 @@ public class MultiThreadedRFileTest {
private Configuration conf = CachedConfiguration.getInstance();
public RFile.Writer writer;
private FSDataOutputStream dos;
- private FSDataInputStream in;
private AccumuloConfiguration accumuloConfiguration;
public Reader reader;
public SortedKeyValueIterator<Key,Value> iter;
[25/45] accumulo git commit: ACCUMULO-4391 removed extra closed check
Posted by ct...@apache.org.
ACCUMULO-4391 removed extra closed check
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3b694418
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3b694418
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3b694418
Branch: refs/heads/1.7
Commit: 3b6944180b4df9d0e56c2296d967b0ec880bd4bd
Parents: 5b08b17
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 21:36:50 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 21:36:50 2016 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +++++++++-----------
1 file changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3b694418/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 60376d5..7fdfc96 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -674,19 +674,17 @@ public final class BCFile {
}
public void finish() throws IOException {
- if (!closed) {
- synchronized (in) {
- if (!closed) {
- try {
- in.close();
- } finally {
- closed = true;
- if (decompressor != null) {
- try {
- compressAlgo.returnDecompressor(decompressor);
- } finally {
- decompressor = null;
- }
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
}
}
}
[31/45] accumulo git commit: Merge branch 'pr-134' into 1.6
Posted by ct...@apache.org.
Merge branch 'pr-134' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9fd94ed4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9fd94ed4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9fd94ed4
Branch: refs/heads/1.8
Commit: 9fd94ed44b342cf4858458f5b923e79c12638844
Parents: 726aa1d 3b69441
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 15:33:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 15:33:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[38/45] accumulo git commit: ACCUMULO-4391 Remove unused field
Posted by ct...@apache.org.
ACCUMULO-4391 Remove unused field
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4c040799
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4c040799
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4c040799
Branch: refs/heads/1.8
Commit: 4c040799f48c21c5f157fabcab8fde5259b6c7ed
Parents: 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:28 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:28 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4c040799/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
index 498551f..a0efeed 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -45,7 +45,6 @@ import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -97,7 +96,6 @@ public class MultiThreadedRFileTest {
private Configuration conf = CachedConfiguration.getInstance();
public RFile.Writer writer;
private FSDataOutputStream dos;
- private FSDataInputStream in;
private AccumuloConfiguration accumuloConfiguration;
public Reader reader;
public SortedKeyValueIterator<Key,Value> iter;
[12/45] accumulo git commit: ACCUMULO-4391 Use a closed flag
Posted by ct...@apache.org.
ACCUMULO-4391 Use a closed flag
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9038d8b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9038d8b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9038d8b3
Branch: refs/heads/1.7
Commit: 9038d8b3a36e7189fbdedfbbb950c7b1f43b2b7d
Parents: 6ff35f0
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Aug 4 00:04:26 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Aug 4 00:04:26 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 31 ++++++++------------
1 file changed, 12 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9038d8b3/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index b182b26..876b585 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
+ private volatile boolean closed = false;
private FSDataInputStream in;
private long pos;
private long end;
@@ -60,12 +61,6 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
- private void check() throws IOException {
- if (in == null) {
- throw new IOException("Stream closed");
- }
- }
-
@Override
public int available() throws IOException {
return (int) (end - pos);
@@ -94,17 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- check(); // ensuring inLocal is not null
- synchronized (inLocal) {
- check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -149,13 +145,10 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- final FSDataInputStream inLocal = in;
- if (inLocal != null) {
- // synchronize on the FSDataInputStream to ensure we block closing if in the read method
- synchronized (inLocal) {
- // Invalidate the state of the stream.
- in = null;
- }
+ // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
}
}
}
[34/45] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ct...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b0981d7d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b0981d7d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b0981d7d
Branch: refs/heads/master
Commit: b0981d7ddf27caf71d3e53519572880262fc6cc3
Parents: 200be5f 9fd94ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 16:55:31 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 16:55:31 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 424 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b0981d7d/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
[23/45] accumulo git commit: Merge branch '1.6' of
https://github.com/ivakegg/accumulo into 1.6
Posted by ct...@apache.org.
Merge branch '1.6' of https://github.com/ivakegg/accumulo into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5b08b17b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5b08b17b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5b08b17b
Branch: refs/heads/master
Commit: 5b08b17bd5d2d528699153adf8bb541402601b4c
Parents: 3c8bc12 feb2a51
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:32:21 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:32:21 2016 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[18/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads. Added a test case which forces at least one of the race conditions.
The remaining conditions have been tested
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads. Added a test case which forces at least one of the race conditions. The remaining conditions have been tested on systems that demonstrated these issues.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3c8bc120
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3c8bc120
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3c8bc120
Branch: refs/heads/1.8
Commit: 3c8bc120b1527eb0cba5390da8e57eb947734af8
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:24:16 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 426 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -615,6 +615,7 @@ public final class BCFile {
private Decompressor decompressor;
private final BlockRegion region;
private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,11 +674,23 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
- private FSDataInputStream in;
+ private volatile boolean closed = false;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -62,12 +63,7 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
+ return (int) (end - pos);
}
@Override
@@ -93,15 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -146,9 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..498551f
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[11/45] accumulo git commit: ACCUMULO-4391 Use a closed flag
Posted by ct...@apache.org.
ACCUMULO-4391 Use a closed flag
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9038d8b3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9038d8b3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9038d8b3
Branch: refs/heads/1.6
Commit: 9038d8b3a36e7189fbdedfbbb950c7b1f43b2b7d
Parents: 6ff35f0
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Aug 4 00:04:26 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Aug 4 00:04:26 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 31 ++++++++------------
1 file changed, 12 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9038d8b3/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index b182b26..876b585 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
+ private volatile boolean closed = false;
private FSDataInputStream in;
private long pos;
private long end;
@@ -60,12 +61,6 @@ class BoundedRangeFileInputStream extends InputStream {
this.mark = -1;
}
- private void check() throws IOException {
- if (in == null) {
- throw new IOException("Stream closed");
- }
- }
-
@Override
public int available() throws IOException {
return (int) (end - pos);
@@ -94,17 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- check(); // ensuring inLocal is not null
- synchronized (inLocal) {
- check(); // ensuring in is not null in which case we were closed which would be followed by someone else reusing the decompressor
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -149,13 +145,10 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- final FSDataInputStream inLocal = in;
- if (inLocal != null) {
- // synchronize on the FSDataInputStream to ensure we block closing if in the read method
- synchronized (inLocal) {
- // Invalidate the state of the stream.
- in = null;
- }
+ // synchronize on the FSDataInputStream to ensure we are blocked if in the read method
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
}
}
}
[20/45] accumulo git commit: ACCUMULO-4391 Added appropriate
synchronization to allow RFile.Reader deepcopies to be used in separate
threads. Added a test case which forces at least one of the race conditions.
The remaining conditions have been tested
Posted by ct...@apache.org.
ACCUMULO-4391 Added appropriate synchronization to allow RFile.Reader deepcopies to be used in separate threads. Added a test case which forces at least one of the race conditions. The remaining conditions have been tested on systems that demonstrated these issues.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3c8bc120
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3c8bc120
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3c8bc120
Branch: refs/heads/1.7
Commit: 3c8bc120b1527eb0cba5390da8e57eb947734af8
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:24:16 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 24 +-
.../bcfile/BoundedRangeFileInputStream.java | 34 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 382 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 426 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
index 3b12d07..be464be 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java
@@ -417,7 +417,10 @@ public class CachableBlockFile {
_bc.close();
if (fin != null) {
- fin.close();
+ // synchronize on the FSDataInputStream to ensure thread safety with the BoundedRangeFileInputStream
+ synchronized (fin) {
+ fin.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
index 4767d91..60376d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BCFile.java
@@ -615,6 +615,7 @@ public final class BCFile {
private Decompressor decompressor;
private final BlockRegion region;
private final InputStream in;
+ private volatile boolean closed;
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, CryptoModule cryptoModule,
Version bcFileVersion, CryptoModuleParameters cryptoParams) throws IOException {
@@ -652,6 +653,7 @@ public final class BCFile {
compressAlgo.returnDecompressor(decompressor);
throw e;
}
+ closed = false;
}
/**
@@ -672,11 +674,23 @@ public final class BCFile {
}
public void finish() throws IOException {
- try {
- in.close();
- } finally {
- compressAlgo.returnDecompressor(decompressor);
- decompressor = null;
+ if (!closed) {
+ synchronized (in) {
+ if (!closed) {
+ try {
+ in.close();
+ } finally {
+ closed = true;
+ if (decompressor != null) {
+ try {
+ compressAlgo.returnDecompressor(decompressor);
+ } finally {
+ decompressor = null;
+ }
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..b5ee61b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.fs.FSDataInputStream;
*/
class BoundedRangeFileInputStream extends InputStream {
- private FSDataInputStream in;
+ private volatile boolean closed = false;
+ private final FSDataInputStream in;
private long pos;
private long end;
private long mark;
@@ -62,12 +63,7 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
-
- return avail;
+ return (int) (end - pos);
}
@Override
@@ -93,15 +89,18 @@ class BoundedRangeFileInputStream extends InputStream {
if (n == 0)
return -1;
Integer ret = 0;
- final FSDataInputStream inLocal = in;
- synchronized (inLocal) {
- inLocal.seek(pos);
+ synchronized (in) {
+ // ensuring we are not closed which would be followed by someone else reusing the decompressor
+ if (closed) {
+ throw new IOException("Stream closed");
+ }
+ in.seek(pos);
try {
ret = AccessController.doPrivileged(new PrivilegedExceptionAction<Integer>() {
@Override
public Integer run() throws IOException {
int ret = 0;
- ret = inLocal.read(b, off, n);
+ ret = in.read(b, off, n);
return ret;
}
});
@@ -146,9 +145,14 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public void close() {
- // Invalidate the state of the stream.
- in = null;
- pos = end;
- mark = -1;
+ // Synchronize on the FSDataInputStream to ensure we are blocked if in the read method:
+ // Once this close completes, the underlying decompression stream may be returned to
+ // the pool and subsequently used. Turns out this is a problem if currently using it to read.
+ if (!closed) {
+ synchronized (in) {
+ // Invalidate the state of the stream.
+ closed = true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..498551f
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Logger LOG = Logger.getLogger(MultiThreadedRFileTest.class);
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ LOG.error(messages.get(message) + ": " + message);
+ LOG.error(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 6a29610..c345065 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -67,8 +67,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -82,11 +80,6 @@ public class RFileTest {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- static {
- Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
- Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
- }
-
static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
public SeekableByteArrayInputStream(byte[] buf) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3c8bc120/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties
index 9f968f8..40adebf 100644
--- a/core/src/test/resources/log4j.properties
+++ b/core/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=ERROR
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=FATAL
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=FATAL
log4j.logger.org.apache.accumulo.core.util.format=FATAL
[22/45] accumulo git commit: Merge branch '1.6' of
https://github.com/ivakegg/accumulo into 1.6
Posted by ct...@apache.org.
Merge branch '1.6' of https://github.com/ivakegg/accumulo into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5b08b17b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5b08b17b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5b08b17b
Branch: refs/heads/1.8
Commit: 5b08b17bd5d2d528699153adf8bb541402601b4c
Parents: 3c8bc12 feb2a51
Author: Ivan Bella <iv...@bella.name>
Authored: Thu Sep 8 12:32:21 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Thu Sep 8 12:32:21 2016 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[02/45] accumulo git commit: ACCUMULO-4391 Added synchronization
around the available() call in the BoundedRangeFileInputStream to allow
RFile.Reader deepcopies to be usable on separate threads. Added a test case
which excersizes RFile.Reader deepcopies
Posted by ct...@apache.org.
ACCUMULO-4391 Added synchronization around the available() call in the BoundedRangeFileInputStream to allow RFile.Reader deepcopies to be usable on separate threads. Added a test case which excersizes RFile.Reader deepcopies across threads.
ACCUMULO-4391: review comments
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5957c223
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5957c223
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5957c223
Branch: refs/heads/1.7
Commit: 5957c2234e9c577ef65c1e0fb44c7c25874059e0
Parents: dcc5dff
Author: Ivan Bella <iv...@bella.name>
Authored: Wed Jul 27 13:11:12 2016 -0400
Committer: Ivan Bella <iv...@bella.name>
Committed: Wed Jul 27 20:25:46 2016 -0400
----------------------------------------------------------------------
.../bcfile/BoundedRangeFileInputStream.java | 13 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 387 +++++++++++++++++++
2 files changed, 395 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
index f93bb84..285040f 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/BoundedRangeFileInputStream.java
@@ -62,12 +62,15 @@ class BoundedRangeFileInputStream extends InputStream {
@Override
public int available() throws IOException {
- int avail = in.available();
- if (pos + avail > end) {
- avail = (int) (end - pos);
- }
+ final FSDataInputStream inLocal = in;
+ synchronized (inLocal) {
+ int avail = inLocal.available();
+ if (pos + avail > end) {
+ avail = (int) (end - pos);
+ }
- return avail;
+ return avail;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5957c223/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
new file mode 100644
index 0000000..662bb1b
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.file.rfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
+import org.apache.accumulo.core.file.rfile.RFile.Reader;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.NamingThreadFactory;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MultiThreadedRFileTest {
+
+ private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+
+ static {
+ Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
+ Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
+ }
+
+ private static void checkIndex(Reader reader) throws IOException {
+ FileSKVIterator indexIter = reader.getIndex();
+
+ if (indexIter.hasTop()) {
+ Key lastKey = new Key(indexIter.getTopKey());
+
+ if (reader.getFirstKey().compareTo(lastKey) > 0)
+ throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey);
+
+ indexIter.next();
+
+ while (indexIter.hasTop()) {
+ if (lastKey.compareTo(indexIter.getTopKey()) > 0)
+ throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey());
+
+ lastKey = new Key(indexIter.getTopKey());
+ indexIter.next();
+
+ }
+
+ if (!reader.getLastKey().equals(lastKey)) {
+ throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey);
+ }
+ }
+ }
+
+ public static class TestRFile {
+
+ private Configuration conf = CachedConfiguration.getInstance();
+ public RFile.Writer writer;
+ private FSDataOutputStream dos;
+ private FSDataInputStream in;
+ private AccumuloConfiguration accumuloConfiguration;
+ public Reader reader;
+ public SortedKeyValueIterator<Key,Value> iter;
+ public File rfile = null;
+ public boolean deepCopy = false;
+
+ public TestRFile(AccumuloConfiguration accumuloConfiguration) {
+ this.accumuloConfiguration = accumuloConfiguration;
+ if (this.accumuloConfiguration == null)
+ this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration();
+ }
+
+ public void close() throws IOException {
+ if (rfile != null) {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ fs.delete(path, false);
+ }
+ }
+
+ public TestRFile deepCopy() throws IOException {
+ TestRFile copy = new TestRFile(accumuloConfiguration);
+ // does not copy any writer resources. This would be for read only.
+ copy.reader = (Reader) reader.deepCopy(null);
+ copy.rfile = rfile;
+ copy.iter = new ColumnFamilySkippingIterator(copy.reader);
+ copy.deepCopy = true;
+
+ checkIndex(copy.reader);
+ return copy;
+ }
+
+ public void openWriter(boolean startDLG) throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deep copy");
+ }
+ if (rfile == null) {
+ rfile = File.createTempFile("TestRFile", ".rf");
+ }
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+ dos = fs.create(path, true);
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf, accumuloConfiguration);
+ writer = new RFile.Writer(_cbw, 1000, 1000);
+
+ if (startDLG)
+ writer.startDefaultLocalityGroup();
+ }
+
+ public void openWriter() throws IOException {
+ openWriter(true);
+ }
+
+ public void closeWriter() throws IOException {
+ if (deepCopy) {
+ throw new IOException("Cannot open writer on a deepcopy");
+ }
+ dos.flush();
+ writer.close();
+ dos.flush();
+ dos.close();
+ }
+
+ public void openReader() throws IOException {
+ FileSystem fs = FileSystem.newInstance(conf);
+ Path path = new Path("file://" + rfile.toString());
+
+ // the caches used to obfuscate the multithreaded issues
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration());
+ reader = new RFile.Reader(_cbr);
+ iter = new ColumnFamilySkippingIterator(reader);
+
+ checkIndex(reader);
+ }
+
+ public void closeReader() throws IOException {
+ reader.close();
+ }
+
+ public void seek(Key nk) throws IOException {
+ iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false);
+ }
+ }
+
+ static Key nk(String row, String cf, String cq, String cv, long ts) {
+ return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts);
+ }
+
+ static Value nv(String val) {
+ return new Value(val.getBytes());
+ }
+
+ public AccumuloConfiguration conf = null;
+
+ @Test
+ public void testMultipleReaders() throws IOException {
+ final List<Throwable> threadExceptions = Collections.synchronizedList(new ArrayList<Throwable>());
+ Map<String,MutableInt> messages = new HashMap<String,MutableInt>();
+ Map<String,String> stackTrace = new HashMap<String,String>();
+
+ final TestRFile trfBase = new TestRFile(conf);
+
+ writeData(trfBase);
+
+ trfBase.openReader();
+
+ try {
+
+ validate(trfBase);
+
+ final TestRFile trfBaseCopy = trfBase.deepCopy();
+
+ validate(trfBaseCopy);
+
+ // now start up multiple RFile deepcopies
+ int maxThreads = 10;
+ String name = "MultiThreadedRFileTestThread";
+ ThreadPoolExecutor pool = new ThreadPoolExecutor(maxThreads + 1, maxThreads + 1, 5 * 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ new NamingThreadFactory(name));
+ pool.allowCoreThreadTimeOut(true);
+ try {
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ TestRFile trf = trfBase;
+ synchronized (trfBaseCopy) {
+ trf = trfBaseCopy.deepCopy();
+ }
+ validate(trf);
+ } catch (Throwable t) {
+ threadExceptions.add(t);
+ }
+ }
+ };
+ for (int i = 0; i < maxThreads; i++) {
+ pool.submit(runnable);
+ }
+ } finally {
+ pool.shutdown();
+ try {
+ pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (Throwable t : threadExceptions) {
+ String msg = t.getClass() + " : " + t.getMessage();
+ if (!messages.containsKey(msg)) {
+ messages.put(msg, new MutableInt(1));
+ } else {
+ messages.get(msg).increment();
+ }
+ StringWriter string = new StringWriter();
+ PrintWriter writer = new PrintWriter(string);
+ t.printStackTrace(writer);
+ writer.flush();
+ stackTrace.put(msg, string.getBuffer().toString());
+ }
+ } finally {
+ trfBase.closeReader();
+ trfBase.close();
+ }
+
+ for (String message : messages.keySet()) {
+ System.out.println(messages.get(message) + ": " + message);
+ System.out.println(stackTrace.get(message));
+ }
+
+ assertTrue(threadExceptions.isEmpty());
+ }
+
+ private void validate(TestRFile trf) throws IOException {
+ Random random = new Random();
+ for (int iteration = 0; iteration < 10; iteration++) {
+ int part = random.nextInt(4);
+
+ Range range = new Range(getKey(part, 0, 0), true, getKey(part, 4, 2048), true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+
+ range = new Range(getKey(4, 4, 0), true, null, true);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ Range range = new Range((Key) null, null);
+ trf.iter.seek(range, EMPTY_COL_FAMS, false);
+
+ Key last = null;
+ for (int part = 0; part < 4; part++) {
+ for (int locality = 0; locality < 4; locality++) {
+ for (int i = 0; i < 2048; i++) {
+ Key key = getKey(part, locality, i);
+ Value value = getValue(i);
+ assertTrue("No record found for row " + part + " locality " + locality + " index " + i, trf.iter.hasTop());
+ assertEquals("Invalid key found for row " + part + " locality " + locality + " index " + i, key, trf.iter.getTopKey());
+ assertEquals("Invalie value found for row " + part + " locality " + locality + " index " + i, value, trf.iter.getTopValue());
+ last = trf.iter.getTopKey();
+ trf.iter.next();
+ }
+ }
+ }
+
+ if (trf.iter.hasTop()) {
+ assertFalse("Found " + trf.iter.getTopKey() + " after " + last + " in " + range, trf.iter.hasTop());
+ }
+ }
+
+ private void writeData(TestRFile trfBase) throws IOException {
+ trfBase.openWriter(false);
+
+ try {
+ for (int locality = 1; locality < 4; locality++) {
+ trfBase.writer.startNewLocalityGroup("locality" + locality, Collections.singleton((ByteSequence) (new ArrayByteSequence(getCf(locality)))));
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, locality, i), getValue(i));
+ }
+ }
+ }
+
+ trfBase.writer.startDefaultLocalityGroup();
+ for (int part = 0; part < 4; part++) {
+ for (int i = 0; i < 2048; i++) {
+ trfBase.writer.append(getKey(part, 0, i), getValue(i));
+ }
+ }
+ } finally {
+ trfBase.closeWriter();
+ }
+ }
+
+ private Key getKey(int part, int locality, int index) {
+ String row = "r000" + part;
+ String cf = getCf(locality);
+ String cq = "cq" + pad(index);
+
+ return nk(row, cf, cq, "", 1);
+ }
+
+ private String pad(int val) {
+ String valStr = String.valueOf(val);
+ switch (valStr.length()) {
+ case 1:
+ return "000" + valStr;
+ case 2:
+ return "00" + valStr;
+ case 3:
+ return "0" + valStr;
+ default:
+ return valStr;
+ }
+ }
+
+ private Value getValue(int index) {
+ return nv("" + index);
+ }
+
+ private String getCf(int locality) {
+ return "cf" + locality;
+ }
+
+}
[41/45] accumulo git commit: Merge branch '1.6' into 1.7
Posted by ct...@apache.org.
Merge branch '1.6' into 1.7
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f21c11cc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f21c11cc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f21c11cc
Branch: refs/heads/1.7
Commit: f21c11cc5a9d896f22ab38f2eb56b4f044a0602b
Parents: b0981d7 4c04079
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:17:57 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:17:57 2016 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
[45/45] accumulo git commit: Merge branch '1.8'
Posted by ct...@apache.org.
Merge branch '1.8'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/aea7cf16
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/aea7cf16
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/aea7cf16
Branch: refs/heads/master
Commit: aea7cf16f40f4666f29d300cb2fafd86f67e3fe9
Parents: 6707120 7dca4ab
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri Sep 9 19:23:01 2016 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri Sep 9 19:23:01 2016 -0400
----------------------------------------------------------------------
.../file/blockfile/impl/CachableBlockFile.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java | 22 +-
.../streams/BoundedRangeFileInputStream.java | 35 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 390 +++++++++++++++++++
.../accumulo/core/file/rfile/RFileTest.java | 7 -
core/src/test/resources/log4j.properties | 2 +
6 files changed, 433 insertions(+), 28 deletions(-)
----------------------------------------------------------------------