You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by vi...@apache.org on 2013/10/31 18:40:14 UTC
git commit: ACCUMULO-1598 Fixes RFlile unit tests
Updated Branches:
refs/heads/master b0da55a65 -> 48fae074d
ACCUMULO-1598 Fixes RFlile unit tests
The RFile tests had several tests in the @Ignore state. Many
of these were redundant, so I simply excise them. There were
a couple of missing files from the repository that the tests
needed; this patch adds them.
Signed-off-by: John Vines <jv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/48fae074
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/48fae074
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/48fae074
Branch: refs/heads/master
Commit: 48fae074de4f029966cbbcb90b1da8dd9ad8016b
Parents: b0da55a
Author: Michael Allen <mi...@sqrrl.com>
Authored: Thu Oct 31 13:17:58 2013 -0400
Committer: John Vines <jv...@gmail.com>
Committed: Thu Oct 31 13:40:09 2013 -0400
----------------------------------------------------------------------
.../accumulo/core/file/rfile/RFileTest.java | 268 -------------------
.../test/resources/crypto-on-accumulo-site.xml | 164 ++++++++++++
...rypto-on-no-key-encryption-accumulo-site.xml | 144 ++++++++++
3 files changed, 308 insertions(+), 268 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48fae074/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index e049531..afc94c6 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -1805,7 +1804,6 @@ public class RFileTest {
}
@Test
- @Ignore
public void testEncryptedRFiles() throws Exception {
String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
@SuppressWarnings("deprecation")
@@ -1831,272 +1829,6 @@ public class RFileTest {
}
@Test
- @Ignore
- public void testRootTabletFromServer() throws Exception {
- String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
- @SuppressWarnings("deprecation")
- AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
- System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
- ((SiteConfiguration) conf).clearAndNull();
-
- TestRFile trf = new TestRFile();
- trf.preGeneratedInputFile = tempFolder.newFile("should_work.rf");
-
- trf.openReader();
- trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
- assertTrue(trf.iter.hasTop());
-
- assertTrue(trf.reader.getLastKey() != null);
-
- trf.closeReader();
-
- if (oldSiteConfigProperty != null) {
- System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
- } else {
- System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
- }
- ((SiteConfiguration) conf).clearAndNull();
-
- }
-
- private static final int TOTAL_NUM_ROWS = 10;
- private static final int ROW_MOD_VALUE = 10;
-
- @Test
- @Ignore
- // These tests will purge the disk cache when the run, so it's not recommended that they be run in development systems.
- public void testEncryptedRFileWriteSpeed() throws Exception {
-
- System.out
- .println("Unencrypted Write, Unencrypted Read (Cache), Unencrypted Read (FS only), Encrypted Write, Encrypted Read (Cache), Encrypted Read (FS Only)");
- int numIterations = 1;
-
- for (int i = 0; i < numIterations; i++) {
- @SuppressWarnings("deprecation")
- AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
- System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_OFF_CONF);
- ((SiteConfiguration) conf).clearAndNull();
-
- TestRFile trf = new TestRFile();
- trf.outputFile = tempFolder.newFile("testUnencryptedRfile.rf");
- trf.openWriter();
-
- double timeTickSize = 1000.0;
- int numRowsRead = 0;
-
- try {
-
- performUnencryptedTests(trf, TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, true);
-
- performEncryptedTests(TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, numRowsRead, false);
-
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
-
- }
-
- }
-
- private void performUnencryptedTests(TestRFile trf, int totalNumRows, int rowModValue, double timeTickSize, boolean first) throws IOException,
- InterruptedException {
- long start = System.currentTimeMillis();
-
- writeRowsToRfile(trf, totalNumRows, rowModValue);
-
- long end = System.currentTimeMillis();
-
- System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
- trf.preGeneratedInputFile = trf.outputFile;
-
- start = System.currentTimeMillis();
-
- trf.openReader();
- trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
- int numRowsRead = 0;
-
- int numRowsToRead = totalNumRows;
- while (numRowsRead < TOTAL_NUM_ROWS) {
- int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
- numRowsToRead -= numRowsReadThisTime;
- numRowsRead += numRowsReadThisTime;
- }
-
- trf.closeReader();
-
- end = System.currentTimeMillis();
-
- System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
- Runtime.getRuntime().exec("purge").waitFor();
-
- start = System.currentTimeMillis();
-
- trf.openReader();
- trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
- numRowsRead = 0;
-
- numRowsToRead = totalNumRows;
- while (numRowsRead < TOTAL_NUM_ROWS) {
- int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
- numRowsToRead -= numRowsReadThisTime;
- numRowsRead += numRowsReadThisTime;
- }
-
- trf.closeReader();
-
- end = System.currentTimeMillis();
-
- if (first) {
- System.out.print("" + ((end - start) / timeTickSize) + ", ");
- } else {
- System.out.println("" + ((end - start) / timeTickSize));
-
- }
-
- }
-
- @SuppressWarnings("deprecation")
- private void performEncryptedTests(int totalNumRows, int rowModValue, double timeTickSize, int numRowsRead, boolean first) throws IOException,
- InterruptedException {
- AccumuloConfiguration conf;
- TestRFile trf;
- long start;
- long end;
- int numRowsToRead;
-
- conf = AccumuloConfiguration.getSiteConfiguration();
- System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
- ((SiteConfiguration) conf).clearAndNull();
-
- trf = new TestRFile();
- trf.outputFile = tempFolder.newFile("testEncryptedRfile.rf");
- trf.openWriter();
-
- start = System.currentTimeMillis();
-
- writeRowsToRfile(trf, totalNumRows, rowModValue);
-
- end = System.currentTimeMillis();
-
- System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
- trf.preGeneratedInputFile = trf.outputFile;
-
- start = System.currentTimeMillis();
-
- trf.openReader();
- trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
-
- numRowsToRead = totalNumRows;
- while (numRowsRead < TOTAL_NUM_ROWS) {
- int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
- numRowsToRead -= numRowsReadThisTime;
- numRowsRead += numRowsReadThisTime;
- }
-
- trf.closeReader();
-
- end = System.currentTimeMillis();
-
- System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
- Runtime.getRuntime().exec("purge").waitFor();
-
- start = System.currentTimeMillis();
-
- trf.openReader();
- trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
- numRowsRead = 0;
-
- numRowsToRead = totalNumRows;
- while (numRowsRead < TOTAL_NUM_ROWS) {
- int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
- numRowsToRead -= numRowsReadThisTime;
- numRowsRead += numRowsReadThisTime;
- }
-
- trf.closeReader();
-
- end = System.currentTimeMillis();
-
- if (first) {
- System.out.print("" + ((end - start) / timeTickSize) + ", ");
- } else {
- System.out.println("" + ((end - start) / timeTickSize));
-
- }
-
- }
-
- private int readRandomRowsFromRfile(TestRFile trf, int totalRowCount, int maxRowsToRead) throws IOException {
- if (maxRowsToRead <= 0) {
- return 0;
- }
-
- int numRowsRead = 0;
- Random rand = new Random(System.nanoTime());
-
- int firstKeyNum = Math.abs(rand.nextInt()) % totalRowCount;
- // int lastKeyNum = Math.abs(rand.nextInt()) % totalRowCount;
- int lastKeyNum = firstKeyNum + 1;
-
- if (lastKeyNum >= totalRowCount) {
- lastKeyNum = firstKeyNum;
- }
-
- if (lastKeyNum < firstKeyNum) {
- int temp = lastKeyNum;
- lastKeyNum = firstKeyNum;
- firstKeyNum = temp;
- }
-
- if (lastKeyNum - firstKeyNum > maxRowsToRead) {
- lastKeyNum = firstKeyNum + maxRowsToRead;
- }
-
- Key firstKey = nk(nf("r_", firstKeyNum), "cf_0", "cq_0", "vis", 0L);
- Key lastKey = nk(nf("r_", lastKeyNum), "cf_19", "cq_19", "vis", 0L);
-
- trf.iter.seek(new Range(firstKey, lastKey), EMPTY_COL_FAMS, false);
- for (int i = firstKeyNum; i < lastKeyNum; i++) {
- @SuppressWarnings("unused")
- Key k = trf.iter.getTopKey();
- @SuppressWarnings("unused")
- Value v = trf.iter.getTopValue();
-
- trf.iter.next();
-
- numRowsRead++;
- }
-
- return numRowsRead;
-
- }
-
- private void writeRowsToRfile(TestRFile trf, int numRowsToWriteAndRead, int rowModValue) throws IOException {
- for (int i = 0; i < numRowsToWriteAndRead; i++) {
- String rowID = nf("r_", (i % rowModValue));
- String colFam = nf("cf_", (i % 20));
- String colQual = nf("cq_", (i % 20));
- String colVis = "vis";
-
- Key k = nk(rowID, colFam, colQual, colVis, i);
- Value v = nv("" + i);
-
- trf.writer.append(k, v);
- }
-
- trf.closeWriter();
- }
-
- @Test
public void testRootTabletEncryption() throws Exception {
// This tests that the normal set of operations used to populate a root tablet
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48fae074/core/src/test/resources/crypto-on-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-on-accumulo-site.xml b/core/src/test/resources/crypto-on-accumulo-site.xml
new file mode 100644
index 0000000..9dc4aac
--- /dev/null
+++ b/core/src/test/resources/crypto-on-accumulo-site.xml
@@ -0,0 +1,164 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!--
+ Put your site-specific accumulo configurations here.
+
+ The available configuration values along with their defaults
+ are documented in docs/config.html
+
+ Unless you are simply testing at your workstation, you will most
+ definitely need to change the three entries below.
+ -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/" values.
+ </description>
+ </property>
+
+ <property>
+ <name>crypto.module.class</name>
+ <value>org.apache.accumulo.core.security.crypto.DefaultCryptoModule</value>
+ </property>
+ <property>
+ <name>crypto.cipher.suite</name>
+ <value>AES/CFB/PKCS5Padding</value>
+ </property>
+ <property>
+ <name>crypto.cipher.algorithm.name</name>
+ <value>AES</value>
+ </property>
+ <property>
+ <name>crypto.cipher.key.length</name>
+ <value>128</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng</name>
+ <value>SHA1PRNG</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng.provider</name>
+ <value>SUN</value>
+ </property>
+ <property>
+ <name>crypto.secret.key.encryption.strategy.class</name>
+ <value>org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy</value>
+ </property>
+ <property>
+ <name>instance.dfs.dir</name>
+ <value>/tmp</value>
+ </property>
+ <property>
+ <name>instance.dfs.uri</name>
+ <value>file:///</value>
+ </property>
+
+ <property>
+ <name>crypto.default.key.strategy.hdfs.uri</name>
+ <value>file:///</value>
+ </property>
+ <property>
+ <name>crypto.default.key.strategy.key.location</name>
+ <value>/tmp/test.secret.key</value>
+ </property>
+
+ <property>
+ <name>crypto.default.key.strategy.cipher.suite</name>
+ <value>AES/ECB/NoPadding</value>
+ </property>
+
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48fae074/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml b/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
new file mode 100644
index 0000000..640abac
--- /dev/null
+++ b/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <!--
+ Put your site-specific accumulo configurations here.
+
+ The available configuration values along with their defaults
+ are documented in docs/config.html
+
+ Unless you are simply testing at your workstation, you will most
+ definitely need to change the three entries below.
+ -->
+
+ <property>
+ <name>instance.zookeeper.host</name>
+ <value>localhost:2181</value>
+ <description>comma separated list of zookeeper servers</description>
+ </property>
+
+ <property>
+ <name>logger.dir.walog</name>
+ <value>walogs</value>
+ <description>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</description>
+ </property>
+
+ <property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+ <description>A secret unique to a given instance that all servers must know in order to communicate with one another.
+ Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+ and then update this file.
+ </description>
+ </property>
+
+ <property>
+ <name>tserver.memory.maps.max</name>
+ <value>80M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.data.size</name>
+ <value>7M</value>
+ </property>
+
+ <property>
+ <name>tserver.cache.index.size</name>
+ <value>20M</value>
+ </property>
+
+ <property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>tserver.sort.buffer.size</name>
+ <value>50M</value>
+ </property>
+
+ <property>
+ <name>tserver.walog.max.size</name>
+ <value>100M</value>
+ </property>
+
+ <property>
+ <name>general.classpaths</name>
+ <value>
+ $ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/proxy/target/classes/,
+ $ACCUMULO_HOME/examples/target/classes/,
+ $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+ </value>
+ <description>Classpaths that accumulo checks for updates and class files.
+ When using the Security Manager, please remove the ".../target/classes/" values.
+ </description>
+ </property>
+
+ <property>
+ <name>crypto.module.class</name>
+ <value>org.apache.accumulo.core.security.crypto.DefaultCryptoModule</value>
+ </property>
+ <property>
+ <name>crypto.cipher.suite</name>
+ <value>AES/CFB/PKCS5Padding</value>
+ </property>
+ <property>
+ <name>crypto.cipher.algorithm.name</name>
+ <value>AES</value>
+ </property>
+ <property>
+ <name>crypto.cipher.key.length</name>
+ <value>128</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng</name>
+ <value>SHA1PRNG</value>
+ </property>
+ <property>
+ <name>crypto.secure.rng.provider</name>
+ <value>SUN</value>
+ </property>
+ <property>
+ <name>instance.dfs.dir</name>
+ <value>/tmp</value>
+ </property>
+ <property>
+ <name>instance.dfs.uri</name>
+ <value>file:///</value>
+ </property>
+
+</configuration>