You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/07/17 04:33:34 UTC
[32/50] [abbrv] ACCUMULO-998 applying Micheal Allen's updated patch
for at-rest encryption
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index a23b6cc..ebdb2d1 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -20,10 +20,14 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
@@ -34,9 +38,13 @@ import java.util.Iterator;
import java.util.Random;
import java.util.Set;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.ArrayByteSequence;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
@@ -46,6 +54,10 @@ import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
import org.apache.accumulo.core.file.rfile.RFile.Reader;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.crypto.CryptoTest;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -58,10 +70,12 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Test;
+
public class RFileTest {
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+
static {
Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF);
Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF);
@@ -150,6 +164,8 @@ public class RFileTest {
public static class TestRFile {
+ public File preGeneratedInputFile = null;
+ public File outputFile = null;
private Configuration conf = CachedConfiguration.getInstance();
public RFile.Writer writer;
private ByteArrayOutputStream baos;
@@ -160,34 +176,56 @@ public class RFileTest {
public SortedKeyValueIterator<Key,Value> iter;
public void openWriter(boolean startDLG) throws IOException {
- baos = new ByteArrayOutputStream();
- dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a"));
+
+ if (outputFile == null) {
+ baos = new ByteArrayOutputStream();
+
+ dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a"));
+ } else {
+ BufferedOutputStream bufos = new BufferedOutputStream(new FileOutputStream(outputFile));
+ dos = new FSDataOutputStream(bufos, new FileSystem.Statistics("a"));
+ }
CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf);
writer = new RFile.Writer(_cbw, 1000, 1000);
if (startDLG)
writer.startDefaultLocalityGroup();
}
-
+
public void openWriter() throws IOException {
openWriter(true);
}
public void closeWriter() throws IOException {
+ dos.flush();
writer.close();
dos.close();
- baos.close();
+ if (baos != null) {
+ baos.close();
+ }
}
public void openReader() throws IOException {
- byte[] data = baos.toByteArray();
+
+ int fileLength = 0;
+ byte[] data = null;
+ if (preGeneratedInputFile != null) {
+ data = new byte[(int) preGeneratedInputFile.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(preGeneratedInputFile));
+ in.readFully(data);
+ in.close();
+ } else {
+ data = baos.toByteArray();
+ }
+
bais = new SeekableByteArrayInputStream(data);
in = new FSDataInputStream(bais);
+ fileLength = data.length;
LruBlockCache indexCache = new LruBlockCache(100000000, 100000);
LruBlockCache dataCache = new LruBlockCache(100000000, 100000);
- CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in, data.length, conf, dataCache, indexCache);
+ CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in, fileLength, conf, dataCache, indexCache);
reader = new RFile.Reader(_cbr);
iter = new ColumnFamilySkippingIterator(reader);
@@ -219,13 +257,13 @@ public class RFileTest {
@Test
public void test1() throws IOException {
- // test an emprt file
+ // test an empty file
TestRFile trf = new TestRFile();
trf.openWriter();
trf.closeWriter();
-
+
trf.openReader();
trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
assertFalse(trf.iter.hasTop());
@@ -537,7 +575,7 @@ public class RFileTest {
@Test
public void test7() throws IOException {
- // these test excercise setting the end key of a range
+ // these tests exercise setting the end key of a range
TestRFile trf = new TestRFile();
@@ -1228,8 +1266,8 @@ public class RFileTest {
count++;
indexIter.next();
}
-
- assert (count > 4);
+
+ assert(count > 4);
trf.iter.seek(new Range(nk("r0000", "cf1", "cq1", "", 1), true, nk("r0001", "cf1", "cq1", "", 1), false), EMPTY_COL_FAMS, false);
@@ -1565,4 +1603,602 @@ public class RFileTest {
reader.close();
}
+
+
+ private AccumuloConfiguration setAndGetAccumuloConfig(String cryptoConfSetting) {
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, cryptoConfSetting);
+ ((SiteConfiguration)conf).clearAndNull();
+ return conf;
+ }
+
+ private void restoreOldConfiguration(String oldSiteConfigProperty, AccumuloConfiguration conf) {
+ if (oldSiteConfigProperty != null) {
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
+ } else {
+ System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ }
+ ((SiteConfiguration)conf).clearAndNull();
+ }
+
+
+ @Test
+ public void testEncRFile1() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test1();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile2() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test2();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile3() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test3();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile4() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test4();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile5() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test5();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile6() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test6();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile7() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test7();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile8() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test8();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile9() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test9();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile10() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test10();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile11() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test11();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+
+ @Test
+ public void testEncRFile12() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test12();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile13() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test13();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile14() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test14();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile16() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test16();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile17() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test17();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile18() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test18();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testEncRFile19() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+
+ test19();
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ //@Test
+ public void testEncryptedRFiles() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
+ ((SiteConfiguration)conf).clearAndNull();
+
+ test1();
+ test2();
+ test3();
+ test4();
+ test5();
+ test6();
+ test7();
+ test8();
+
+
+ if (oldSiteConfigProperty != null) {
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
+ } else {
+ System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ }
+ ((SiteConfiguration)conf).clearAndNull();
+ }
+
+ //@Test
+ public void testRootTabletFromServer() throws Exception {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
+ ((SiteConfiguration)conf).clearAndNull();
+
+ TestRFile trf = new TestRFile();
+ trf.preGeneratedInputFile = new File("/tmp/should_work.rf");
+
+ trf.openReader();
+ trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
+ assert(trf.iter.hasTop());
+
+ assert(trf.reader.getLastKey() != null);
+
+ trf.closeReader();
+
+
+ if (oldSiteConfigProperty != null) {
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
+ } else {
+ System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ }
+ ((SiteConfiguration)conf).clearAndNull();
+
+ }
+
+ private static final int TOTAL_NUM_ROWS = 10;
+ private static final int ROW_MOD_VALUE = 10;
+
+ //@Test
+ // These tests will purge the disk cache when the run, so it's not recommended that they be run in development systems.
+ public void testEncryptedRFileWriteSpeed() throws Exception {
+
+
+ System.out.println("Unencrypted Write, Unencrypted Read (Cache), Unencrypted Read (FS only), Encrypted Write, Encrypted Read (Cache), Encrypted Read (FS Only)");
+ int numIterations = 1;
+
+ for (int i = 0; i < numIterations; i++) {
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_OFF_CONF);
+ ((SiteConfiguration)conf).clearAndNull();
+
+ TestRFile trf = new TestRFile();
+ trf.outputFile = new File("/tmp/testUnencryptedRfile.rf");
+ trf.openWriter();
+
+
+
+ double timeTickSize = 1000.0;
+ int numRowsRead = 0;
+
+
+ try {
+
+ performUnencryptedTests(trf, TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, true);
+
+ performEncryptedTests(TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, numRowsRead, false);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+
+
+
+ }
+
+ }
+
+ private void performUnencryptedTests(TestRFile trf, int totalNumRows, int rowModValue, double timeTickSize, boolean first) throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+
+
+ writeRowsToRfile(trf, totalNumRows, rowModValue);
+
+ long end = System.currentTimeMillis();
+
+ System.out.print(""+((end - start) / timeTickSize) + ", ");
+
+ trf.preGeneratedInputFile = trf.outputFile;
+
+ start = System.currentTimeMillis();
+
+ trf.openReader();
+ trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
+ int numRowsRead = 0;
+
+ int numRowsToRead = totalNumRows;
+ while (numRowsRead < TOTAL_NUM_ROWS) {
+ int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
+
+ numRowsToRead -= numRowsReadThisTime;
+ numRowsRead += numRowsReadThisTime;
+ }
+
+ trf.closeReader();
+
+ end = System.currentTimeMillis();
+
+ System.out.print(""+((end - start) / timeTickSize) + ", ");
+
+ Runtime.getRuntime().exec("purge").waitFor();
+
+ start = System.currentTimeMillis();
+
+ trf.openReader();
+ trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
+ numRowsRead = 0;
+
+ numRowsToRead = totalNumRows;
+ while (numRowsRead < TOTAL_NUM_ROWS) {
+ int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
+
+ numRowsToRead -= numRowsReadThisTime;
+ numRowsRead += numRowsReadThisTime;
+ }
+
+ trf.closeReader();
+
+ end = System.currentTimeMillis();
+
+
+ if (first) {
+ System.out.print(""+((end - start) / timeTickSize)+", ");
+ } else {
+ System.out.println(""+((end - start) / timeTickSize));
+
+ }
+
+
+
+ //trf.outputFile.delete();
+ }
+
+ @SuppressWarnings("deprecation")
+ private void performEncryptedTests(int totalNumRows, int rowModValue, double timeTickSize, int numRowsRead, boolean first) throws IOException, InterruptedException {
+ AccumuloConfiguration conf;
+ TestRFile trf;
+ long start;
+ long end;
+ int numRowsToRead;
+
+ conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
+ ((SiteConfiguration)conf).clearAndNull();
+
+ trf = new TestRFile();
+ trf.outputFile = new File("/tmp/testEncryptedRfile.rf");
+ trf.openWriter();
+
+ start = System.currentTimeMillis();
+
+ writeRowsToRfile(trf, totalNumRows, rowModValue);
+
+ end = System.currentTimeMillis();
+
+ System.out.print(""+((end - start) / timeTickSize) + ", ");
+
+ trf.preGeneratedInputFile = trf.outputFile;
+
+ start = System.currentTimeMillis();
+
+ trf.openReader();
+ trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
+
+ numRowsToRead = totalNumRows;
+ while (numRowsRead < TOTAL_NUM_ROWS) {
+ int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
+
+ numRowsToRead -= numRowsReadThisTime;
+ numRowsRead += numRowsReadThisTime;
+ }
+
+ trf.closeReader();
+
+ end = System.currentTimeMillis();
+
+ System.out.print(""+((end - start) / timeTickSize)+", ");
+
+ Runtime.getRuntime().exec("purge").waitFor();
+
+ start = System.currentTimeMillis();
+
+ trf.openReader();
+ trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
+ numRowsRead = 0;
+
+ numRowsToRead = totalNumRows;
+ while (numRowsRead < TOTAL_NUM_ROWS) {
+ int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
+
+ numRowsToRead -= numRowsReadThisTime;
+ numRowsRead += numRowsReadThisTime;
+ }
+
+ trf.closeReader();
+
+ end = System.currentTimeMillis();
+
+ if (first) {
+ System.out.print(""+((end - start) / timeTickSize)+", ");
+ } else {
+ System.out.println(""+((end - start) / timeTickSize));
+
+ }
+
+
+ trf.outputFile.delete();
+ }
+
+ private int readRandomRowsFromRfile(TestRFile trf, int totalRowCount, int maxRowsToRead) throws IOException {
+ if (maxRowsToRead <= 0) {
+ return 0;
+ }
+
+ int numRowsRead = 0;
+ Random rand = new Random(System.nanoTime());
+
+ int firstKeyNum = Math.abs(rand.nextInt()) % totalRowCount;
+ //int lastKeyNum = Math.abs(rand.nextInt()) % totalRowCount;
+ int lastKeyNum = firstKeyNum + 1;
+
+ if (lastKeyNum >= totalRowCount) {
+ lastKeyNum = firstKeyNum;
+ }
+
+ if (lastKeyNum < firstKeyNum) {
+ int temp = lastKeyNum;
+ lastKeyNum = firstKeyNum;
+ firstKeyNum = temp;
+ }
+
+ if (lastKeyNum - firstKeyNum > maxRowsToRead) {
+ lastKeyNum = firstKeyNum + maxRowsToRead;
+ }
+
+ Key firstKey = nk(nf("r_", firstKeyNum), "cf_0", "cq_0", "vis", 0L);
+ Key lastKey = nk(nf("r_", lastKeyNum), "cf_19", "cq_19", "vis", 0L);
+
+ trf.iter.seek(new Range(firstKey, lastKey), EMPTY_COL_FAMS, false);
+ for (int i = firstKeyNum; i < lastKeyNum; i++) {
+ @SuppressWarnings("unused")
+ Key k = trf.iter.getTopKey();
+ @SuppressWarnings("unused")
+ Value v = trf.iter.getTopValue();
+
+ trf.iter.next();
+
+ numRowsRead++;
+ }
+
+ return numRowsRead;
+
+ }
+
+ private void writeRowsToRfile(TestRFile trf, int numRowsToWriteAndRead, int rowModValue) throws IOException {
+ for (int i = 0; i < numRowsToWriteAndRead; i++) {
+ String rowID = nf("r_", (i % rowModValue));
+ String colFam = nf("cf_", (i % 20));
+ String colQual = nf("cq_", (i % 20));
+ String colVis = "vis";
+
+ Key k = nk(rowID, colFam, colQual, colVis, i);
+ Value v = nv(""+i);
+
+ trf.writer.append(k, v);
+ }
+
+ trf.closeWriter();
+ }
+
+
+ @Test
+ public void testRootTabletEncryption() throws Exception {
+
+ // This tests that the normal set of operations used to populate a root tablet
+
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
+ ((SiteConfiguration)conf).clearAndNull();
+
+ // populate the root tablet with info about the default tablet
+ // the root tablet contains the key extent and locations of all the
+ // metadata tablets
+ //String initRootTabFile = ServerConstants.getMetadataTableDir() + "/root_tablet/00000_00000."
+ // + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
+ //FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, fs, conf, AccumuloConfiguration.getDefaultConfiguration());
+
+ TestRFile testRfile = new TestRFile();
+ testRfile.openWriter();
+
+ RFile.Writer mfw = testRfile.writer;
+
+ // mfw.startDefaultLocalityGroup();
+
+ //mfw.startDefaultLocalityGroup();
+
+ Text tableExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), MetadataSchema.TabletsSection.getRange().getEndKey().getRow()));
+
+ // table tablet's directory
+ Key tableDirKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0);
+ mfw.append(tableDirKey, new Value(/*TABLE_TABLETS_TABLET_DIR*/"/table_info".getBytes()));
+
+ // table tablet time
+ Key tableTimeKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0);
+ mfw.append(tableTimeKey, new Value((/*TabletTime.LOGICAL_TIME_ID*/ 'L' + "0").getBytes()));
+
+ // table tablet's prevrow
+ Key tablePrevRowKey = new Key(tableExtent, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(),
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0);
+ mfw.append(tablePrevRowKey, KeyExtent.encodePrevEndRow(null));
+
+ // ----------] default tablet info
+ Text defaultExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), null));
+
+ // default's directory
+ Key defaultDirKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0);
+ mfw.append(defaultDirKey, new Value(Constants.DEFAULT_TABLET_LOCATION.getBytes()));
+
+ // default's time
+ Key defaultTimeKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(),
+ TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0);
+ mfw.append(defaultTimeKey, new Value((/*TabletTime.LOGICAL_TIME_ID*/ 'L' + "0").getBytes()));
+
+ // default's prevrow
+ Key defaultPrevRowKey = new Key(defaultExtent, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(),
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0);
+ mfw.append(defaultPrevRowKey, KeyExtent.encodePrevEndRow(MetadataSchema.TabletsSection.getRange().getEndKey().getRow()));
+
+
+ testRfile.closeWriter();
+
+ if (true) {
+ FileOutputStream fileOutputStream = new FileOutputStream(new File("/tmp/testEncryptedRootFile.rf"));
+ fileOutputStream.write(testRfile.baos.toByteArray());
+ fileOutputStream.flush();
+ fileOutputStream.close();
+ }
+
+
+
+
+ testRfile.openReader();
+ testRfile.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
+ assert(testRfile.iter.hasTop());
+
+ assert(testRfile.reader.getLastKey() != null);
+
+
+
+
+ testRfile.closeReader();
+
+ if (oldSiteConfigProperty != null) {
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
+ } else {
+ System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ }
+ ((SiteConfiguration)conf).clearAndNull();
+
+ }
}
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java b/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java
new file mode 100644
index 0000000..d37a7e0
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/security/crypto/CryptoTest.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.security.crypto;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.InvalidKeyException;
+import java.security.Key;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
+import java.util.Arrays;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class CryptoTest {
+
+ private static final int MARKER_INT = 0xCADEFEDD;
+ private static final String MARKER_STRING = "1 2 3 a b c";
+ public static final String CONFIG_FILE_SYSTEM_PROP = "org.apache.accumulo.config.file";
+ public static final String CRYPTO_ON_CONF = "crypto-on-accumulo-site.xml";
+ public static final String CRYPTO_OFF_CONF = "crypto-off-accumulo-site.xml";
+ public static final String CRYPTO_ON_KEK_OFF_CONF = "crypto-on-no-key-encryption-accumulo-site.xml";
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testNoCryptoStream() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_OFF_CONF);
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ assertNotNull(params);
+ assertEquals("NullCipher", params.getAlgorithmName());
+ assertNull(params.getEncryptionMode());
+ assertNull(params.getPadding());
+
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+ assertNotNull(cryptoModule);
+ assertTrue(cryptoModule instanceof CryptoModuleFactory.NullCryptoModule);
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ params.setPlaintextOutputStream(out);
+
+ params = cryptoModule.getEncryptingOutputStream(params);
+ assertNotNull(params.getEncryptedOutputStream());
+ assertEquals(out, params.getEncryptedOutputStream());
+
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testCryptoModuleParamsParsing() {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ assertNotNull(params);
+ assertEquals("AES", params.getAlgorithmName());
+ assertEquals("CFB", params.getEncryptionMode());
+ assertEquals("PKCS5Padding", params.getPadding());
+ assertEquals(128, params.getKeyLength());
+ assertEquals("SHA1PRNG", params.getRandomNumberGenerator());
+ assertEquals("SUN", params.getRandomNumberGeneratorProvider());
+ assertEquals("org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy", params.getKeyEncryptionStrategyClass());
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testCryptoModuleParamsValidation1() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
+
+ try {
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+
+ assertTrue(cryptoModule instanceof DefaultCryptoModule);
+
+ exception.expect(RuntimeException.class);
+ cryptoModule.getEncryptingOutputStream(params);
+
+
+ } finally {
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+ }
+
+ @Test
+ public void testCryptoModuleParamsValidation2() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
+
+ try {
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+
+ assertTrue(cryptoModule instanceof DefaultCryptoModule);
+
+ exception.expect(RuntimeException.class);
+ cryptoModule.getDecryptingInputStream(params);
+ } finally {
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+ }
+
+ private String getStringifiedBytes(String s) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ DataOutputStream dataOut = new DataOutputStream(out);
+
+ dataOut.writeUTF(s);
+ dataOut.close();
+ byte[] stringMarkerBytes = out.toByteArray();
+ return Arrays.toString(stringMarkerBytes);
+
+ }
+
+ private String getStringifiedBytes(int i) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ DataOutputStream dataOut = new DataOutputStream(out);
+
+ dataOut.writeInt(i);
+ dataOut.close();
+ byte[] stringMarkerBytes = out.toByteArray();
+ return Arrays.toString(stringMarkerBytes);
+
+ }
+
+ @Test
+ public void testCryptoModuleBasicReadWrite() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_KEK_OFF_CONF);
+
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ assertTrue(cryptoModule instanceof DefaultCryptoModule);
+ assertTrue(params.getKeyEncryptionStrategyClass() == null || params.getKeyEncryptionStrategyClass().equals(""));
+
+ byte[] resultingBytes = setUpSampleEncryptedBytes(cryptoModule, params);
+
+ // If we get here, we have encrypted bytes
+ ByteArrayInputStream in = new ByteArrayInputStream(resultingBytes);
+
+ params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setEncryptedInputStream(in);
+
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ InputStream plaintextIn = params.getPlaintextInputStream();
+
+ assertNotNull(plaintextIn);
+ assertTrue(plaintextIn != in);
+ DataInputStream dataIn = new DataInputStream(plaintextIn);
+ String markerString = dataIn.readUTF();
+ int markerInt = dataIn.readInt();
+
+ assertEquals(MARKER_STRING, markerString);
+ assertEquals(MARKER_INT, markerInt);
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ private byte[] setUpSampleEncryptedBytes(CryptoModule cryptoModule, CryptoModuleParameters params) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ params.setPlaintextOutputStream(out);
+
+ params = cryptoModule.getEncryptingOutputStream(params);
+
+ assertNotNull(params.getEncryptedOutputStream());
+ assertTrue(params.getEncryptedOutputStream() != out);
+
+ DataOutputStream dataOut = new DataOutputStream(params.getEncryptedOutputStream());
+ dataOut.writeUTF(MARKER_STRING);
+ dataOut.writeInt(MARKER_INT);
+ dataOut.close();
+
+ byte[] resultingBytes = out.toByteArray();
+ String stringifiedBytes = Arrays.toString(resultingBytes);
+
+ String stringifiedMarkerBytes = getStringifiedBytes(MARKER_STRING);
+ String stringifiedOtherBytes = getStringifiedBytes(MARKER_INT);
+
+
+ // OK, let's make sure it's encrypted
+ assertTrue(!stringifiedBytes.contains(stringifiedMarkerBytes));
+ assertTrue(!stringifiedBytes.contains(stringifiedOtherBytes));
+ return resultingBytes;
+ }
+
+ @Test
+ public void testKeyEncryptionAndCheckThatFileCannotBeReadWithoutKEK() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
+
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ assertTrue(cryptoModule instanceof DefaultCryptoModule);
+ assertNotNull(params.getKeyEncryptionStrategyClass());
+ assertEquals("org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy", params.getKeyEncryptionStrategyClass());
+
+ byte[] resultingBytes = setUpSampleEncryptedBytes(cryptoModule, params);
+
+ // So now that we have bytes encrypted by a key encrypted to a KEK, turn off the KEK configuration and try
+ // to decrypt. We expect this to fail. This also tests our ability to override the key encryption strategy.
+ conf = setAndGetAccumuloConfig(CRYPTO_ON_KEK_OFF_CONF);
+ params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setOverrideStreamsSecretKeyEncryptionStrategy(true);
+
+ ByteArrayInputStream in = new ByteArrayInputStream(resultingBytes);
+ params.setEncryptedInputStream(in);
+
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ assertNotNull(params.getPlaintextInputStream());
+ DataInputStream dataIn = new DataInputStream(params.getPlaintextInputStream());
+ // We expect the following operation to fail and throw an exception
+ try {
+ exception.expect(IOException.class);
+ @SuppressWarnings("unused")
+ String markerString = dataIn.readUTF();
+ }
+ finally {
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+ }
+
+ @Test
+ public void testKeyEncryptionNormalPath() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
+
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ assertTrue(cryptoModule instanceof DefaultCryptoModule);
+ assertNotNull(params.getKeyEncryptionStrategyClass());
+ assertEquals("org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy", params.getKeyEncryptionStrategyClass());
+
+ byte[] resultingBytes = setUpSampleEncryptedBytes(cryptoModule, params);
+
+ params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setOverrideStreamsSecretKeyEncryptionStrategy(true);
+
+ ByteArrayInputStream in = new ByteArrayInputStream(resultingBytes);
+ params.setEncryptedInputStream(in);
+
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ assertNotNull(params.getPlaintextInputStream());
+ DataInputStream dataIn = new DataInputStream(params.getPlaintextInputStream());
+
+ String markerString = dataIn.readUTF();
+ int markerInt = dataIn.readInt();
+
+ assertEquals(MARKER_STRING, markerString);
+ assertEquals(MARKER_INT, markerInt);
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ @Test
+ public void testChangingCryptoParamsAndCanStillDecryptPreviouslyEncryptedFiles() throws IOException {
+ String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ AccumuloConfiguration conf = setAndGetAccumuloConfig(CRYPTO_ON_CONF);
+
+ CryptoModule cryptoModule = CryptoModuleFactory.getCryptoModule(conf);
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+
+ assertTrue(cryptoModule instanceof DefaultCryptoModule);
+ assertNotNull(params.getKeyEncryptionStrategyClass());
+ assertEquals("org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy", params.getKeyEncryptionStrategyClass());
+
+ byte[] resultingBytes = setUpSampleEncryptedBytes(cryptoModule, params);
+
+ // Now we're going to create a params object and set its algorithm and key length different
+ // from those configured within the site configuration. After doing this, we should
+ // still be able to read the file that was created with a different set of parameters.
+ params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setAlgorithmName("DESede");
+ params.setKeyLength(24 * 8);
+
+ ByteArrayInputStream in = new ByteArrayInputStream(resultingBytes);
+ params.setEncryptedInputStream(in);
+
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ assertNotNull(params.getPlaintextInputStream());
+ DataInputStream dataIn = new DataInputStream(params.getPlaintextInputStream());
+ String markerString = dataIn.readUTF();
+ int markerInt = dataIn.readInt();
+
+ assertEquals(MARKER_STRING, markerString);
+ assertEquals(MARKER_INT, markerInt);
+
+ restoreOldConfiguration(oldSiteConfigProperty, conf);
+ }
+
+ private void restoreOldConfiguration(String oldSiteConfigProperty, AccumuloConfiguration conf) {
+ if (oldSiteConfigProperty != null) {
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
+ } else {
+ System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
+ }
+ ((SiteConfiguration)conf).clearAndNull();
+ }
+
+
+
+ private AccumuloConfiguration setAndGetAccumuloConfig(String cryptoConfSetting) {
+ @SuppressWarnings("deprecation")
+ AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
+ System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, cryptoConfSetting);
+ ((SiteConfiguration)conf).clearAndNull();
+ return conf;
+ }
+
+ @Test
+ public void testKeyWrapAndUnwrap() throws NoSuchAlgorithmException, NoSuchPaddingException, NoSuchProviderException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
+ Cipher keyWrapCipher = Cipher.getInstance("AES/ECB/NoPadding");
+ SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN");
+
+ byte[] kek = new byte[16];
+ random.nextBytes(kek);
+ byte[] randomKey = new byte[16];
+ random.nextBytes(randomKey);
+
+ keyWrapCipher.init(Cipher.WRAP_MODE, new SecretKeySpec(kek, "AES"));
+
+ Key randKey = new SecretKeySpec(randomKey, "AES");
+
+ byte[] wrappedKey = keyWrapCipher.wrap(randKey);
+
+ assert(wrappedKey != null);
+ assert(wrappedKey.length == randomKey.length);
+
+
+ Cipher keyUnwrapCipher = Cipher.getInstance("AES/ECB/NoPadding");
+ keyUnwrapCipher.init(Cipher.UNWRAP_MODE, new SecretKeySpec(kek, "AES"));
+ Key unwrappedKey = keyUnwrapCipher.unwrap(wrappedKey, "AES", Cipher.SECRET_KEY);
+
+ byte[] unwrappedKeyBytes = unwrappedKey.getEncoded();
+ assert(Arrays.equals(unwrappedKeyBytes, randomKey));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/core/src/test/resources/crypto-off-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-off-accumulo-site.xml b/core/src/test/resources/crypto-off-accumulo-site.xml
new file mode 100644
index 0000000..667e9a3
--- /dev/null
+++ b/core/src/test/resources/crypto-off-accumulo-site.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!--
+ Put your site-specific accumulo configurations here.
+
+ The available configuration values along with their defaults
+ are documented in docs/config.html
+
+ Unless you are simply testing at your workstation, you will most
+ definitely need to change the three entries below.
+ -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/" values.
+ </description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/core/src/test/resources/crypto-on-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-on-accumulo-site.xml b/core/src/test/resources/crypto-on-accumulo-site.xml
new file mode 100644
index 0000000..9dc4aac
--- /dev/null
+++ b/core/src/test/resources/crypto-on-accumulo-site.xml
@@ -0,0 +1,164 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!--
+ Put your site-specific accumulo configurations here.
+
+ The available configuration values along with their defaults
+ are documented in docs/config.html
+
+ Unless you are simply testing at your workstation, you will most
+ definitely need to change the three entries below.
+ -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/" values.
+ </description>
+ </property>
+
+ <property>
+ <name>crypto.module.class</name>
+ <value>org.apache.accumulo.core.security.crypto.DefaultCryptoModule</value>
+ </property>
+ <property>
+ <name>crypto.cipher.suite</name>
+ <value>AES/CFB/PKCS5Padding</value>
+ </property>
+ <property>
+ <name>crypto.cipher.algorithm.name</name>
+ <value>AES</value>
+ </property>
+ <property>
+ <name>crypto.cipher.key.length</name>
+ <value>128</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng</name>
+ <value>SHA1PRNG</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng.provider</name>
+ <value>SUN</value>
+ </property>
+ <property>
+ <name>crypto.secret.key.encryption.strategy.class</name>
+ <value>org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy</value>
+ </property>
+ <property>
+ <name>instance.dfs.dir</name>
+ <value>/tmp</value>
+ </property>
+ <property>
+ <name>instance.dfs.uri</name>
+ <value>file:///</value>
+ </property>
+
+ <property>
+ <name>crypto.default.key.strategy.hdfs.uri</name>
+ <value>file:///</value>
+ </property>
+ <property>
+ <name>crypto.default.key.strategy.key.location</name>
+ <value>/tmp/test.secret.key</value>
+ </property>
+
+ <property>
+ <name>crypto.default.key.strategy.cipher.suite</name>
+ <value>AES/ECB/NoPadding</value>
+ </property>
+
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml b/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
new file mode 100644
index 0000000..640abac
--- /dev/null
+++ b/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!--
+ Put your site-specific accumulo configurations here.
+
+ The available configuration values along with their defaults
+ are documented in docs/config.html
+
+ Unless you are simply testing at your workstation, you will most
+ definitely need to change the three entries below.
+ -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/" values.
+ </description>
+ </property>
+
+ <property>
+ <name>crypto.module.class</name>
+ <value>org.apache.accumulo.core.security.crypto.DefaultCryptoModule</value>
+ </property>
+ <property>
+ <name>crypto.cipher.suite</name>
+ <value>AES/CFB/PKCS5Padding</value>
+ </property>
+ <property>
+ <name>crypto.cipher.algorithm.name</name>
+ <value>AES</value>
+ </property>
+ <property>
+ <name>crypto.cipher.key.length</name>
+ <value>128</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng</name>
+ <value>SHA1PRNG</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng.provider</name>
+ <value>SUN</value>
+ </property>
+ <property>
+ <name>instance.dfs.dir</name>
+ <value>/tmp</value>
+ </property>
+ <property>
+ <name>instance.dfs.uri</name>
+ <value>file:///</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 37791e4..0dc56a4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -142,7 +142,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.4</version>
+ <version>1.7</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
index 21bef0f..6d574b9 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
@@ -40,6 +40,8 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.crypto.CryptoModuleFactory;
+import org.apache.accumulo.core.security.crypto.CryptoModuleParameters;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.server.ServerConstants;
@@ -271,22 +273,26 @@ public class DfsLogger {
}
// Initialize the crypto operations.
- @SuppressWarnings("deprecation")
org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf
.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
// Initialize the log file with a header and the crypto params used to set up this log file.
logFile.write(LOG_FILE_HEADER_V2.getBytes());
- Map<String,String> cryptoOpts = conf.getConfiguration().getAllPropertiesWithPrefix(Property.CRYPTO_PREFIX);
+
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
+
+ params.setPlaintextOutputStream(logFile);
+
+ // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
+ // so that that crypto module can re-read its own parameters.
+
+ logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
- logFile.writeInt(cryptoOpts.size());
- for (String key : cryptoOpts.keySet()) {
- logFile.writeUTF(key);
- logFile.writeUTF(cryptoOpts.get(key));
- }
- @SuppressWarnings("deprecation")
- OutputStream encipheringOutputStream = cryptoModule.getEncryptingOutputStream(logFile, cryptoOpts);
+ //@SuppressWarnings("deprecation")
+ //OutputStream encipheringOutputStream = cryptoModule.getEncryptingOutputStream(logFile, cryptoOpts);
+ params = cryptoModule.getEncryptingOutputStream(params);
+ OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
// If the module just kicks back our original stream, then just use it, don't wrap it in
// another data OutputStream.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65b5a3a3/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
index ea6c1ce..7518edb 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/LogSorter.java
@@ -19,8 +19,8 @@ package org.apache.accumulo.server.tabletserver.log;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -34,6 +34,8 @@ import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.master.thrift.RecoveryStatus;
+import org.apache.accumulo.core.security.crypto.CryptoModuleFactory;
+import org.apache.accumulo.core.security.crypto.CryptoModuleParameters;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.core.zookeeper.ZooUtil;
@@ -111,45 +113,44 @@ public class LogSorter {
fs.deleteRecursively(new Path(destPath));
FSDataInputStream tmpInput = fs.open(srcPath);
- DataInputStream tmpDecryptingInput = tmpInput;
-
- Map<String,String> cryptoOpts = new HashMap<String,String>();
- tmpInput = DfsLogger.readHeader(fs, srcPath, cryptoOpts);
-
- if (!cryptoOpts.containsKey(Property.CRYPTO_MODULE_CLASS.getKey())) {
-
- log.debug("Log file " + name + " not encrypted");
-
+
+ byte[] magic = DfsLogger.LOG_FILE_HEADER_V2.getBytes();
+ byte[] magicBuffer = new byte[magic.length];
+ tmpInput.readFully(magicBuffer);
+ if (!Arrays.equals(magicBuffer, magic)) {
+ tmpInput.seek(0);
synchronized (this) {
- this.input = tmpInput;
- this.decryptingInput = tmpInput;
+ this.input = tmpInput;
+ this.decryptingInput = tmpInput;
}
-
} else {
+ // We read the crypto module class name here because we need to boot strap the class. The class itself will read any
+ // additional parameters it needs from the underlying stream.
+ String cryptoModuleClassname = tmpInput.readUTF();
+ org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory
+ .getCryptoModule(cryptoModuleClassname);
- String cryptoModuleName = cryptoOpts.get(Property.CRYPTO_MODULE_CLASS.getKey());
- if (cryptoModuleName == null) {
- // If for whatever reason we didn't get a configured crypto module (old log file version, for instance)
- // default to using the default configuration entry (usually NullCipher).
- cryptoModuleName = AccumuloConfiguration.getDefaultConfiguration().get(Property.CRYPTO_MODULE_CLASS);
- }
+ // Create the parameters and set the input stream into those parameters
+ CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf);
+ params.setEncryptedInputStream(tmpInput);
+ // Create the plaintext input stream from the encrypted one
+ params = cryptoModule.getDecryptingInputStream(params);
+
+ // Store the plaintext input stream into member variables
synchronized (this) {
this.input = tmpInput;
+
+ if (params.getPlaintextInputStream() instanceof DataInputStream) {
+ this.decryptingInput = (DataInputStream)params.getPlaintextInputStream();
+ } else {
+ this.decryptingInput = new DataInputStream(params.getPlaintextInputStream());
+ }
+
}
- @SuppressWarnings("deprecation")
- org.apache.accumulo.core.security.crypto.CryptoModule cryptoOps = org.apache.accumulo.core.security.crypto.CryptoModuleFactory
- .getCryptoModule(cryptoModuleName);
- @SuppressWarnings("deprecation")
- InputStream decryptingInputStream = cryptoOps.getDecryptingInputStream(input, cryptoOpts);
-
- tmpDecryptingInput = new DataInputStream(decryptingInputStream);
-
- synchronized (this) {
- this.decryptingInput = tmpDecryptingInput;
- }
}
+
final long bufferSize = conf.getMemoryInBytes(Property.TSERV_SORT_BUFFER_SIZE);
Thread.currentThread().setName("Sorting " + name + " for recovery");