You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2018/10/13 17:33:04 UTC
[accumulo-testing] branch master updated: fixed peformance test
(#39)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push:
new 5355941 fixed peformance test (#39)
5355941 is described below
commit 53559410596e8d8cc79c5148232ee44e79fecfa4
Author: Keith Turner <kt...@apache.org>
AuthorDate: Sat Oct 13 13:33:00 2018 -0400
fixed peformance test (#39)
---
bin/performance-test | 4 ++--
conf/cluster-control.sh.uno | 18 ++++++++++++---
.../testing/core/continuous/ContinuousIngest.java | 7 +++---
.../testing/core/continuous/UndefinedAnalyzer.java | 2 +-
.../testing/core/ingest/BulkImportDirectory.java | 2 +-
.../testing/core/performance/PerformanceTest.java | 2 +-
.../core/performance/SystemConfiguration.java | 2 +-
.../core/performance/impl/MergeSiteConfig.java | 26 +++++++++-------------
.../performance/tests/RandomCachedLookupsPT.java | 2 +-
.../core/performance/tests/ScanExecutorPT.java | 5 +++--
.../core/performance/tests/ScanFewFamiliesPT.java | 2 +-
11 files changed, 41 insertions(+), 31 deletions(-)
diff --git a/bin/performance-test b/bin/performance-test
index b606ad3..7fa1e68 100755
--- a/bin/performance-test
+++ b/bin/performance-test
@@ -74,9 +74,9 @@ case "$1" in
echo "Running test $test_class"
pt_tmp=$(mktemp -d -t accumulo_pt_XXXXXXX)
setup_accumulo
- get_config_file accumulo-site.xml "$pt_tmp"
+ get_config_file accumulo.properties "$pt_tmp"
CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.MergeSiteConfig "$test_class" "$pt_tmp"
- put_config_file "$pt_tmp/accumulo-site.xml"
+ put_config_file "$pt_tmp/accumulo.properties"
put_server_code "$at_home/core/target/accumulo-testing-core-$at_version.jar"
start_accumulo
get_config_file accumulo-client.properties "$pt_tmp"
diff --git a/conf/cluster-control.sh.uno b/conf/cluster-control.sh.uno
index 2b9abdb..678ee35 100644
--- a/conf/cluster-control.sh.uno
+++ b/conf/cluster-control.sh.uno
@@ -30,21 +30,33 @@ function get_version {
ACCUMULO)
(
# run following in sub shell so it does not pollute
- . $UNO_HOME/conf/uno.conf
+ if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+ . $UNO_HOME/conf/uno-local.conf
+ else
+ . $UNO_HOME/conf/uno.conf
+ fi
echo $ACCUMULO_VERSION
)
;;
HADOOP)
(
# run following in sub shell so it does not pollute
- . $UNO_HOME/conf/uno.conf
+ if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+ . $UNO_HOME/conf/uno-local.conf
+ else
+ . $UNO_HOME/conf/uno.conf
+ fi
echo $HADOOP_VERSION
)
;;
ZOOKEEPER)
(
# run following in sub shell so it does not pollute
- . $UNO_HOME/conf/uno.conf
+ if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+ . $UNO_HOME/conf/uno-local.conf
+ else
+ . $UNO_HOME/conf/uno.conf
+ fi
echo $ZOOKEEPER_VERSION
)
;;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
index 499be09..375c619 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
@@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
-import com.google.common.base.Preconditions;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
@@ -36,14 +35,16 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.trace.CountSampler;
import org.apache.accumulo.core.trace.Trace;
+import org.apache.accumulo.core.trace.TraceSamplers;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.testing.core.TestProps;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
public class ContinuousIngest {
private static final Logger log = LoggerFactory.getLogger(ContinuousIngest.class);
@@ -129,7 +130,7 @@ public class ContinuousIngest {
}
BatchWriter bw = client.createBatchWriter(tableName);
- bw = Trace.wrapAll(bw, new CountSampler(1024));
+ bw = Trace.wrapAll(bw, TraceSamplers.countSampler(1024));
Random r = new Random();
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
index 8c4c14d..56b7aa0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
@@ -36,12 +36,12 @@ import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.server.cli.ClientOnDefaultTable;
import org.apache.hadoop.io.Text;
import com.beust.jcommander.Parameter;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java b/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
index e5617c4..bc1b305 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
@@ -18,10 +18,10 @@ package org.apache.accumulo.testing.core.ingest;
import java.io.IOException;
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.server.cli.ClientOnRequiredTable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
index 04ef98f..d957286 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
@@ -19,7 +19,7 @@ package org.apache.accumulo.testing.core.performance;
public interface PerformanceTest {
- SystemConfiguration getConfiguration();
+ SystemConfiguration getSystemConfig();
Report runTest(Environment env) throws Exception;
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
index 60f02e5..c626a01 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
@@ -31,7 +31,7 @@ public class SystemConfiguration {
return this;
}
- public Map<String,String> getAccumuloSite() {
+ public Map<String,String> getAccumuloConfig() {
return accumuloSite;
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
index 2f79a88..aeacf20 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
@@ -17,36 +17,32 @@
package org.apache.accumulo.testing.core.performance.impl;
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.Reader;
+import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Properties;
import org.apache.accumulo.testing.core.performance.PerformanceTest;
-import org.apache.hadoop.conf.Configuration;
public class MergeSiteConfig {
public static void main(String[] args) throws Exception {
String className = args[0];
- Path confFile = Paths.get(args[1], "accumulo-site.xml");
+ Path confFile = Paths.get(args[1], "accumulo.properties");
PerformanceTest perfTest = Class.forName(className).asSubclass(PerformanceTest.class).newInstance();
- Configuration conf = new Configuration(false);
- byte[] newConf;
+ Properties props = new Properties();
-
- try(BufferedInputStream in = new BufferedInputStream(Files.newInputStream(confFile))){
- conf.addResource(in);
- perfTest.getConfiguration().getAccumuloSite().forEach((k,v) -> conf.set(k, v));
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- conf.writeXml(baos);
- baos.close();
- newConf = baos.toByteArray();
+ try(Reader in = Files.newBufferedReader(confFile)){
+ props.load(in);
}
+ perfTest.getSystemConfig().getAccumuloConfig().forEach((k,v) -> props.setProperty(k, v));
- Files.write(confFile, newConf);
+ try(Writer out = Files.newBufferedWriter(confFile)){
+ props.store(out, "Modified by performance test");
+ }
}
}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
index 28d34bb..5655c3f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
@@ -59,7 +59,7 @@ public class RandomCachedLookupsPT implements PerformanceTest {
private static final int NUM_ROWS = 100000;
@Override
- public SystemConfiguration getConfiguration() {
+ public SystemConfiguration getSystemConfig() {
Map<String,String> siteCfg = new HashMap<>();
siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "1000");
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
index 994af06..8f1e2d0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
@@ -64,7 +64,7 @@ public class ScanExecutorPT implements PerformanceTest {
+ "a dedicated executor. If the scan prioritizer or dispatcher is not working properly, " + "then the short scans will be orders of magnitude slower.";
@Override
- public SystemConfiguration getConfiguration() {
+ public SystemConfiguration getSystemConfig() {
Map<String,String> siteCfg = new HashMap<>();
siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "200");
@@ -85,6 +85,7 @@ public class ScanExecutorPT implements PerformanceTest {
Map<String,String> props = new HashMap<>();
props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "executor", "se1");
props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "heed_hints", "true");
+ props.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
env.getClient().tableOperations().create(tableName,
new NewTableConfiguration().setProperties(props));
@@ -100,7 +101,7 @@ public class ScanExecutorPT implements PerformanceTest {
TestExecutor<Long> longScans = startLongScans(env, tableName, stop);
LongSummaryStatistics shortStats1 = runShortScans(env, tableName, 50000);
- LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 500000);
+ LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 100000);
stop.set(true);
long t4 = System.currentTimeMillis();
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
index b098ba0..4f675f5 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
@@ -45,7 +45,7 @@ public class ScanFewFamiliesPT implements PerformanceTest {
private static final int NUM_QUALS = 1;
@Override
- public SystemConfiguration getConfiguration() {
+ public SystemConfiguration getSystemConfig() {
return new SystemConfiguration();
}