You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2019/04/24 22:10:18 UTC
[accumulo] branch master updated: fixes #1111 support reading
unencrypted WALs from 1.9 (#1127)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new be3a6b7 fixes #1111 support reading unencrypted WALs from 1.9 (#1127)
be3a6b7 is described below
commit be3a6b7654e6fd95c69a502121e7513db81e9140
Author: Keith Turner <kt...@apache.org>
AuthorDate: Wed Apr 24 18:10:13 2019 -0400
fixes #1111 support reading unencrypted WALs from 1.9 (#1127)
---
.../org/apache/accumulo/tserver/log/DfsLogger.java | 38 +++++++++++++++-------
.../java/org/apache/accumulo/test/TestIngest.java | 8 +++--
.../org/apache/accumulo/test/VerifyIngest.java | 1 +
3 files changed, 34 insertions(+), 13 deletions(-)
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index c994f19..46a119d 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -27,6 +27,7 @@ import static org.apache.accumulo.tserver.logger.LogEvents.OPEN;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Method;
@@ -84,9 +85,9 @@ import com.google.common.base.Preconditions;
*
*/
public class DfsLogger implements Comparable<DfsLogger> {
- // older versions should no longer be supported in 2.0
- public static final String LOG_FILE_HEADER_V2 = "--- Log File Header (v2) ---";
+ // older version supported for upgrade
public static final String LOG_FILE_HEADER_V3 = "--- Log File Header (v3) ---";
+
/**
* Simplified encryption technique supported in V4.
*
@@ -361,11 +362,17 @@ public class DfsLogger implements Comparable<DfsLogger> {
AccumuloConfiguration conf) throws IOException {
DataInputStream decryptingInput;
- byte[] magic = DfsLogger.LOG_FILE_HEADER_V4.getBytes(UTF_8);
- byte[] magicBuffer = new byte[magic.length];
+ byte[] magic4 = DfsLogger.LOG_FILE_HEADER_V4.getBytes(UTF_8);
+ byte[] magic3 = DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8);
+
+ if (magic4.length != magic3.length)
+ throw new AssertionError("Always expect log file headers to be same length : " + magic4.length
+ + " != " + magic3.length);
+
+ byte[] magicBuffer = new byte[magic4.length];
try {
input.readFully(magicBuffer);
- if (Arrays.equals(magicBuffer, magic)) {
+ if (Arrays.equals(magicBuffer, magic4)) {
byte[] params = CryptoUtils.readParams(input);
CryptoService cryptoService =
CryptoServiceFactory.newInstance(conf, ClassloaderType.ACCUMULO);
@@ -375,14 +382,23 @@ public class DfsLogger implements Comparable<DfsLogger> {
log.debug("Using {} for decrypting WAL", cryptoService.getClass().getSimpleName());
decryptingInput = cryptoService instanceof NoCryptoService ? input
: new DataInputStream(decrypter.decryptStream(input));
- } else {
- log.error("Unsupported WAL version.");
- input.seek(0);
+ } else if (Arrays.equals(magicBuffer, magic3)) {
+ // Read logs files from Accumulo 1.9
+ String cryptoModuleClassname = input.readUTF();
+ if (!cryptoModuleClassname.equals("NullCryptoModule")) {
+ throw new IllegalArgumentException(
+ "Old encryption modules not supported at this time. Unsupported module : "
+ + cryptoModuleClassname);
+ }
+
decryptingInput = input;
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported write ahead log version " + new String(magicBuffer));
}
- } catch (Exception e) {
- log.warn("Got EOFException trying to read WAL header information,"
- + " assuming the rest of the file has no data.");
+ } catch (EOFException e) {
+ // Explicitly catch any exceptions that should be converted to LogHeaderIncompleteException
+
// A TabletServer might have died before the (complete) header was written
throw new LogHeaderIncompleteException(e);
}
diff --git a/test/src/main/java/org/apache/accumulo/test/TestIngest.java b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
index dd32acc..9b29e4f 100644
--- a/test/src/main/java/org/apache/accumulo/test/TestIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestIngest.java
@@ -142,8 +142,7 @@ public class TestIngest {
description = "place columns in this column family", converter = VisibilityConverter.class)
ColumnVisibility columnVisibility = new ColumnVisibility();
- public IngestParams getIngestPrams() {
- IngestParams params = new IngestParams(getClientProps(), tableName);
+ protected void populateIngestPrams(IngestParams params) {
params.createTable = createTable;
params.numsplits = numsplits;
params.startRow = startRow;
@@ -157,6 +156,11 @@ public class TestIngest {
params.stride = stride;
params.columnFamily = columnFamily;
params.columnVisibility = columnVisibility;
+ }
+
+ public IngestParams getIngestPrams() {
+ IngestParams params = new IngestParams(getClientProps(), tableName);
+ populateIngestPrams(params);
return params;
}
}
diff --git a/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java b/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
index 6aaf267..2cdd164 100644
--- a/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
+++ b/test/src/main/java/org/apache/accumulo/test/VerifyIngest.java
@@ -81,6 +81,7 @@ public class VerifyIngest {
public VerifyParams getVerifyParams() {
VerifyParams params = new VerifyParams(getClientProps(), tableName);
+ populateIngestPrams(params);
params.useGet = useGet;
return params;
}