You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by gj...@apache.org on 2019/09/17 17:52:44 UTC
[phoenix] branch 4.x-HBase-1.3 updated: PHOENIX-4285 Add
PHERF.LOG_PER_NROWS constraint to PHERF.properties file
This is an automated email from the ASF dual-hosted git repository.
gjacoby pushed a commit to branch 4.x-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.3 by this push:
new 6d9e2a8 PHOENIX-4285 Add PHERF.LOG_PER_NROWS constraint to PHERF.properties file
6d9e2a8 is described below
commit 6d9e2a8b92e36f4702627f5ca926bf5a961f51e2
Author: Xinyi <xy...@salesforce.com>
AuthorDate: Tue Sep 10 16:26:22 2019 -0700
PHOENIX-4285 Add PHERF.LOG_PER_NROWS constraint to PHERF.properties file
Signed-off-by: Geoffrey Jacoby <gj...@apache.org>
---
phoenix-pherf/config/pherf.properties | 3 ++
.../main/java/org/apache/phoenix/pherf/Pherf.java | 26 ++++++++++++++
.../org/apache/phoenix/pherf/PherfConstants.java | 1 +
.../phoenix/pherf/workload/WriteWorkload.java | 8 ++++-
.../java/org/apache/phoenix/pherf/PherfTest.java | 42 ++++++++++++++++++++++
.../org/apache/phoenix/pherf/ResourceTest.java | 1 +
6 files changed, 80 insertions(+), 1 deletion(-)
diff --git a/phoenix-pherf/config/pherf.properties b/phoenix-pherf/config/pherf.properties
index bfce8bc..9892e2e 100644
--- a/phoenix-pherf/config/pherf.properties
+++ b/phoenix-pherf/config/pherf.properties
@@ -21,6 +21,9 @@ pherf.default.threadpool=10
# Interval in Ms that the JMX monitors will take a snapshot and dump to log
pherf.default.monitorFrequency=30000
+# Default value to display log line after every 'N' row load
+pherf.default.log_per_nrows=1000000
+
# Default number of writers to use when loading data
# 0 - Set the number of writers to use all available cores
# 1-N - ANy integer value for the number of threads to use
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
index 05e747a..2572a34 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
@@ -75,6 +75,8 @@ public class Pherf {
options.addOption("rowCountOverride", true,
"Row count override to use instead of one specified in scenario.");
options.addOption("hint", true, "Executes all queries with specified hint. Example SMALL");
+ options.addOption("log_per_nrows", true,
+ "Default value to display log line after every 'N' row load");
options.addOption("diff", false,
"Run pherf in verification mode and diff with exported results");
options.addOption("export", false,
@@ -143,6 +145,8 @@ public class Pherf {
LOGGER.debug("Using Monitor: " + monitor);
LOGGER.debug("Monitor Frequency Ms:" + monitorFrequency);
+ properties.setProperty(PherfConstants.LOG_PER_NROWS_NAME, getLogPerNRow(command));
+
preLoadData = command.hasOption("l");
executeQuerySets = command.hasOption("q");
zookeeper = command.getOptionValue("z", "localhost");
@@ -193,6 +197,28 @@ public class Pherf {
ResultUtil.setFileSuffix(label);
}
+ private String getLogPerNRow(CommandLine command) {
+ try {
+ String logPerNRows = (command.hasOption("log_per_nrows")) ?
+ command.getOptionValue("log_per_nrows") :
+ properties.getProperty(
+ PherfConstants.LOG_PER_NROWS_NAME,
+ String.valueOf(PherfConstants.LOG_PER_NROWS)
+ );
+ if (Integer.valueOf(logPerNRows) > 0) {
+ return logPerNRows;
+ }
+ } catch (Exception e) {
+ LOGGER.warn("Invalid Log per N rows value. Phoenix will pick the default value.");
+ }
+
+ return String.valueOf(PherfConstants.LOG_PER_NROWS);
+ }
+
+ public Properties getProperties() {
+ return this.properties;
+ }
+
public static void main(String[] args) {
try {
new Pherf(args).run();
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
index e7ba056..2a7a9b9 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
@@ -67,6 +67,7 @@ public class PherfConstants {
// TODO MOve to properties
// log out data load per n rows
+ public static final String LOG_PER_NROWS_NAME = "pherf.default.log_per_nrows";
public static final int LOG_PER_NROWS = 1000000;
public static final String COMBINED_FILE_NAME = "COMBINED";
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index 98029c0..a498bfe 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -266,6 +266,12 @@ public class WriteWorkload implements Workload {
long maxDuration = (WriteWorkload.this.writeParams == null) ? Long.MAX_VALUE :
WriteWorkload.this.writeParams.getExecutionDurationInMs();
+ int logPerNRows = PherfConstants.LOG_PER_NROWS;
+ String customizedLogPerNRows = connection.getClientInfo().
+ getProperty(PherfConstants.LOG_PER_NROWS_NAME);
+ if (customizedLogPerNRows!= null) {
+ logPerNRows = Integer.valueOf(customizedLogPerNRows);
+ }
last = start = System.currentTimeMillis();
String sql = buildSql(columns, tableName);
stmt = connection.prepareStatement(sql);
@@ -298,7 +304,7 @@ public class WriteWorkload implements Workload {
+ " rows for this thread (" + this.hashCode() + ") in ("
+ duration + ") Ms");
- if (i % PherfConstants.LOG_PER_NROWS == 0 && i != 0) {
+ if (i % logPerNRows == 0 && i != 0) {
dataLoadThreadTime.add(tableName,
Thread.currentThread().getName(), i,
System.currentTimeMillis() - logStartTime);
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
index 80ab971..6a24ced 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
@@ -22,6 +22,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.ExpectedSystemExit;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
public class PherfTest {
@Rule
public final ExpectedSystemExit exit = ExpectedSystemExit.none();
@@ -40,4 +44,42 @@ public class PherfTest {
exit.expectSystemExitWithStatus(1);
Pherf.main(args);
}
+
+ @Test
+ public void testDefaultLogPerNRowsArgument() throws Exception {
+ String[] args = {"-listFiles"};
+ assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS),
+ getLogPerNRowsValue(new Pherf(args).getProperties()));
+ }
+
+ @Test
+ public void testCustomizedLogPerNRowsArgument() throws Exception {
+ Long customizedPerNRows = 15l;
+ String[] args = {"-listFiles", "-log_per_nrows", customizedPerNRows.toString()};
+ assertEquals(customizedPerNRows,
+ getLogPerNRowsValue(new Pherf(args).getProperties()));
+ }
+
+ @Test
+ public void testInvalidLogPerNRowsArgument() throws Exception {
+ Long zero = 0l;
+ Long negativeOne = -1l;
+ String invaildNum = "abc";
+
+ String[] args = {"-listFiles", "-log_per_nrows", zero.toString()};
+ assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS),
+ getLogPerNRowsValue(new Pherf(args).getProperties()));
+
+ String[] args2 = {"-listFiles", "-log_per_nrows", negativeOne.toString()};
+ assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS),
+ getLogPerNRowsValue(new Pherf(args2).getProperties()));
+
+ String[] args3 = {"-listFiles", "-log_per_nrows", invaildNum};
+ assertEquals(Long.valueOf(PherfConstants.LOG_PER_NROWS),
+ getLogPerNRowsValue(new Pherf(args3).getProperties()));
+ }
+
+ private Long getLogPerNRowsValue(Properties prop) {
+ return Long.valueOf(prop.getProperty(PherfConstants.LOG_PER_NROWS_NAME));
+ }
}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java
index d8cad48..5920025 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResourceTest.java
@@ -51,6 +51,7 @@ public class ResourceTest {
PherfConstants constants = PherfConstants.create();
assertNotNull(constants.getProperty("pherf.default.dataloader.threadpool"));
assertNotNull(constants.getProperty("pherf.default.results.dir"));
+ assertNotNull(constants.getProperty(PherfConstants.LOG_PER_NROWS_NAME));
}
private Collection<Path> assertResources(String pattern, String rootDir, String assertStr) throws Exception {