You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2014/04/18 16:43:01 UTC
[1/6] git commit: Merge branch '1.6.0-SNAPSHOT' of
https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.0-SNAPSHOT
Repository: accumulo
Updated Branches:
refs/heads/master 8aedbe6ca -> 94008f6a1
Merge branch '1.6.0-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cacb1b62
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cacb1b62
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cacb1b62
Branch: refs/heads/master
Commit: cacb1b625bf7db27ba70afc805b11fc730532744
Parents: 9fb7417 4048bb2
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Apr 16 15:16:39 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Apr 16 15:16:39 2014 -0400
----------------------------------------------------------------------
README | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
[4/6] git commit: ACCUMULO-2690 expanding test to validate giant read
as well
Posted by vi...@apache.org.
ACCUMULO-2690 expanding test to validate giant read as well
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8d6432be
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8d6432be
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8d6432be
Branch: refs/heads/master
Commit: 8d6432bec3d479eb15c3609a66b588ccfc7e5308
Parents: 8e3dc7b
Author: John Vines <vi...@apache.org>
Authored: Thu Apr 17 18:41:39 2014 -0400
Committer: John Vines <vi...@apache.org>
Committed: Thu Apr 17 18:41:39 2014 -0400
----------------------------------------------------------------------
.../core/security/crypto/BlockedIOStreamTest.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8d6432be/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index a116110..17a20da 100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.core.security.crypto;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -136,7 +137,16 @@ public class BlockedIOStreamTest {
baos.close();
int blocks = (int) Math.ceil(size / (blockSize - 4.0));
- assertEquals(blocks * 16, baos.toByteArray().length);
+ byte[] byteStream = baos.toByteArray();
+
+ assertEquals(blocks * 16, byteStream.length);
+
+ DataInputStream blockIn = new DataInputStream(new BlockedInputStream(new ByteArrayInputStream(byteStream), blockSize, blockSize));
+ byte[] giantRead = new byte[size];
+ blockIn.readFully(giantRead, 0, size);
+ blockIn.close();
+
+ assertArrayEquals(giant, giantRead);
}
}
[6/6] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by vi...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Conflicts:
core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/94008f6a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/94008f6a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/94008f6a
Branch: refs/heads/master
Commit: 94008f6a1235edf178294effc7623576a249cdb4
Parents: 8aedbe6 0550720
Author: John Vines <vi...@apache.org>
Authored: Fri Apr 18 10:42:51 2014 -0400
Committer: John Vines <vi...@apache.org>
Committed: Fri Apr 18 10:42:51 2014 -0400
----------------------------------------------------------------------
.../security/crypto/BlockedOutputStream.java | 17 +++---
.../security/crypto/NoFlushOutputStream.java | 10 +--
.../security/crypto/BlockedIOStreamTest.java | 64 +++++++++++++++++---
.../apache/accumulo/tserver/log/DfsLogger.java | 9 ++-
4 files changed, 72 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94008f6a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index 8ea55a6,212aaaf..d80124c
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@@ -16,15 -16,18 +16,17 @@@
*/
package org.apache.accumulo.core.security.crypto;
-import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+ import java.util.Arrays;
import java.util.Random;
-import org.apache.accumulo.core.Constants;
-import org.apache.commons.lang.ArrayUtils;
import org.junit.Test;
public class BlockedIOStreamTest {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94008f6a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
[3/6] git commit: ACCUMULO-2690 fixing output stream short circuiting
in walog
Posted by vi...@apache.org.
ACCUMULO-2690 fixing output stream short circuiting in walog
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8e3dc7b4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8e3dc7b4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8e3dc7b4
Branch: refs/heads/master
Commit: 8e3dc7b475b6b803c4c4402e54ad185cf60b3d89
Parents: 1734489
Author: John Vines <vi...@apache.org>
Authored: Thu Apr 17 18:00:49 2014 -0400
Committer: John Vines <vi...@apache.org>
Committed: Thu Apr 17 18:00:49 2014 -0400
----------------------------------------------------------------------
.../core/security/crypto/NoFlushOutputStream.java | 10 ++--------
.../java/org/apache/accumulo/tserver/log/DfsLogger.java | 9 ++++++---
2 files changed, 8 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e3dc7b4/core/src/main/java/org/apache/accumulo/core/security/crypto/NoFlushOutputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/security/crypto/NoFlushOutputStream.java b/core/src/main/java/org/apache/accumulo/core/security/crypto/NoFlushOutputStream.java
index 2f9f4bb..17fc06a 100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/NoFlushOutputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/NoFlushOutputStream.java
@@ -16,22 +16,16 @@
*/
package org.apache.accumulo.core.security.crypto;
-import java.io.FilterOutputStream;
-import java.io.IOException;
+import java.io.DataOutputStream;
import java.io.OutputStream;
-public class NoFlushOutputStream extends FilterOutputStream {
+public class NoFlushOutputStream extends DataOutputStream {
public NoFlushOutputStream(OutputStream out) {
super(out);
}
@Override
- public void write(byte[] b, int off, int len) throws IOException {
- out.write(b, off, len);
- }
-
- @Override
public void flush() {}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8e3dc7b4/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index c960bd6..eb04f09 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -368,7 +368,8 @@ public class DfsLogger {
CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
- params.setPlaintextOutputStream(new NoFlushOutputStream(logFile));
+ NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
+ params.setPlaintextOutputStream(nfos);
// In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
// so that that crypto module can re-read its own parameters.
@@ -380,9 +381,11 @@ public class DfsLogger {
// If the module just kicks back our original stream, then just use it, don't wrap it in
// another data OutputStream.
- if (encipheringOutputStream == logFile) {
- encryptingLogFile = logFile;
+ if (encipheringOutputStream == nfos) {
+ log.debug("No enciphering, using raw output stream");
+ encryptingLogFile = nfos;
} else {
+ log.debug("Enciphering found, wrapping in DataOutputStream");
encryptingLogFile = new DataOutputStream(encipheringOutputStream);
}
[2/6] git commit: ACCUMULO-2671 Refactoring BlockedOutputStream to
not recurse. With test
Posted by vi...@apache.org.
ACCUMULO-2671 Refactoring BlockedOutputStream to not recurse. With test
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/17344890
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/17344890
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/17344890
Branch: refs/heads/master
Commit: 173448903a4b3eb5e755421428e65a4a4dd67de5
Parents: cacb1b6
Author: John Vines <vi...@apache.org>
Authored: Thu Apr 17 17:57:32 2014 -0400
Committer: John Vines <vi...@apache.org>
Committed: Thu Apr 17 17:57:32 2014 -0400
----------------------------------------------------------------------
.../security/crypto/BlockedOutputStream.java | 17 ++++----
.../security/crypto/BlockedIOStreamTest.java | 44 +++++++++++++++-----
2 files changed, 44 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/17344890/core/src/main/java/org/apache/accumulo/core/security/crypto/BlockedOutputStream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/security/crypto/BlockedOutputStream.java b/core/src/main/java/org/apache/accumulo/core/security/crypto/BlockedOutputStream.java
index ca72055..3ce648e 100644
--- a/core/src/main/java/org/apache/accumulo/core/security/crypto/BlockedOutputStream.java
+++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/BlockedOutputStream.java
@@ -72,15 +72,18 @@ public class BlockedOutputStream extends OutputStream {
@Override
public void write(byte b[], int off, int len) throws IOException {
- if (bb.remaining() >= len) {
- bb.put(b, off, len);
- if (bb.remaining() == 0)
- flush();
- } else {
+ // Can't recurse here in case the len is large and the blocksize is small (and the stack is small)
+ // So we'll just fill up the buffer over and over
+ while (len >= bb.remaining()) {
int remaining = bb.remaining();
- write(b, off, remaining);
- write(b, off + remaining, len - remaining);
+ bb.put(b, off, remaining);
+ // This is guaranteed to have the buffer filled, so we'll just flush it. No check needed
+ flush();
+ off += remaining;
+ len -= remaining;
}
+ // And then write the remainder (and this is guaranteed to not fill the buffer, so we won't flush afteward
+ bb.put(b, off, len);
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/17344890/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index b344fc3..a116110 100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@ -70,49 +70,73 @@ public class BlockedIOStreamTest {
@Test
public void testSmallBufferBlockedIO() throws IOException {
- writeRead(16, (12 + 4) * (int) (Math.ceil(25.0/12) + Math.ceil(31.0/12)));
+ writeRead(16, (12 + 4) * (int) (Math.ceil(25.0 / 12) + Math.ceil(31.0 / 12)));
}
-
+
@Test
public void testSpillingOverOutputStream() throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
// buffer will be size 12
BlockedOutputStream blockOut = new BlockedOutputStream(baos, 16, 16);
Random r = new Random(22);
-
+
byte[] undersized = new byte[11];
byte[] perfectSized = new byte[12];
byte[] overSized = new byte[13];
byte[] perfectlyOversized = new byte[13];
byte filler = (byte) r.nextInt();
-
+
r.nextBytes(undersized);
r.nextBytes(perfectSized);
r.nextBytes(overSized);
r.nextBytes(perfectlyOversized);
-
+
// 1 block
blockOut.write(undersized);
blockOut.write(filler);
blockOut.flush();
-
+
// 2 blocks
blockOut.write(perfectSized);
blockOut.write(filler);
blockOut.flush();
-
+
// 2 blocks
blockOut.write(overSized);
blockOut.write(filler);
blockOut.flush();
-
+
// 3 blocks
blockOut.write(undersized);
blockOut.write(perfectlyOversized);
blockOut.write(filler);
blockOut.flush();
-
+
+ blockOut.close();
+ assertEquals(16 * 8, baos.toByteArray().length);
+ }
+
+ @Test
+ public void testGiantWrite() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int blockSize = 16;
+ // buffer will be size 12
+ BlockedOutputStream blockOut = new BlockedOutputStream(baos, blockSize, blockSize);
+ Random r = new Random(22);
+
+ int size = 1024 * 1024 * 128;
+ byte[] giant = new byte[size];
+
+ r.nextBytes(giant);
+
+ blockOut.write(giant);
+ blockOut.flush();
+
blockOut.close();
- assertEquals(16*8, baos.toByteArray().length);
+ baos.close();
+
+ int blocks = (int) Math.ceil(size / (blockSize - 4.0));
+ assertEquals(blocks * 16, baos.toByteArray().length);
}
+
}
[5/6] git commit: ACCUMULO-2671 adjusting test to make jenkins happy
Posted by vi...@apache.org.
ACCUMULO-2671 adjusting test to make jenkins happy
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/05507206
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/05507206
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/05507206
Branch: refs/heads/master
Commit: 05507206410bc289da98bfb7931d1cea43cc69a2
Parents: 8d6432b
Author: John Vines <vi...@apache.org>
Authored: Fri Apr 18 10:40:34 2014 -0400
Committer: John Vines <vi...@apache.org>
Committed: Fri Apr 18 10:40:34 2014 -0400
----------------------------------------------------------------------
.../security/crypto/BlockedIOStreamTest.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05507206/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
index 17a20da..212aaaf 100644
--- a/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/BlockedIOStreamTest.java
@@ -23,9 +23,11 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Random;
import org.apache.accumulo.core.Constants;
+import org.apache.commons.lang.ArrayUtils;
import org.junit.Test;
public class BlockedIOStreamTest {
@@ -127,8 +129,13 @@ public class BlockedIOStreamTest {
int size = 1024 * 1024 * 128;
byte[] giant = new byte[size];
+ byte[] pattern = new byte[1024];
- r.nextBytes(giant);
+ r.nextBytes(pattern);
+
+ for (int i = 0; i < size / 1024; i++) {
+ System.arraycopy(pattern, 0, giant, i * 1024, 1024);
+ }
blockOut.write(giant);
blockOut.flush();
@@ -142,11 +149,15 @@ public class BlockedIOStreamTest {
assertEquals(blocks * 16, byteStream.length);
DataInputStream blockIn = new DataInputStream(new BlockedInputStream(new ByteArrayInputStream(byteStream), blockSize, blockSize));
- byte[] giantRead = new byte[size];
- blockIn.readFully(giantRead, 0, size);
+ Arrays.fill(giant, (byte) 0);
+ blockIn.readFully(giant, 0, size);
blockIn.close();
- assertArrayEquals(giant, giantRead);
+ for (int i = 0; i < size / 1024; i++) {
+ byte[] readChunk = new byte[1024];
+ System.arraycopy(giant, i * 1024, readChunk, 0, 1024);
+ assertArrayEquals(pattern, readChunk);
+ }
}
}