You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2015/08/12 03:40:02 UTC
[10/12] phoenix git commit: PHOENIX-2145 Pherf - Make update stats
optional and fix threads not exiting after performance run
PHOENIX-2145 Pherf - Make update stats optional and fix threads not exiting after performance run
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/060a00ca
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/060a00ca
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/060a00ca
Branch: refs/heads/4.5-HBase-0.98
Commit: 060a00ca50f84da561a7c583f55e22bca5d26641
Parents: 0fcefaa
Author: Mujtaba <mu...@apache.org>
Authored: Tue Aug 4 15:47:44 2015 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Aug 11 18:36:52 2015 -0700
----------------------------------------------------------------------
phoenix-pherf/pom.xml | 3 --
.../org/apache/phoenix/pherf/DataIngestIT.java | 4 +-
.../src/main/assembly/components-minimal.xml | 2 +-
.../java/org/apache/phoenix/pherf/Pherf.java | 23 +++++------
.../apache/phoenix/pherf/PherfConstants.java | 5 +++
.../apache/phoenix/pherf/util/PhoenixUtil.java | 11 +++++
.../phoenix/pherf/workload/QueryExecutor.java | 3 +-
.../phoenix/pherf/workload/WriteWorkload.java | 43 ++++++++++----------
.../org/apache/phoenix/pherf/PherfTest.java | 2 +-
9 files changed, 55 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index 68a8905..6cfb0e8 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -108,9 +108,6 @@
</resource>
<resource>
<directory>config</directory>
- <includes>
- <include>**/*.properties</include>
- </includes>
</resource>
</resources>
<testResources>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
index 8a340b3..1defbb1 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
@@ -20,6 +20,8 @@ package org.apache.phoenix.pherf;
import com.jcabi.jdbc.JdbcSession;
import com.jcabi.jdbc.Outcome;
+
+import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
import org.apache.phoenix.pherf.configuration.Column;
import org.apache.phoenix.pherf.configuration.DataModel;
import org.apache.phoenix.pherf.configuration.DataTypeMapping;
@@ -66,7 +68,7 @@ public class DataIngestIT extends ResultBaseTestIT {
scenario.getTableNameWithoutSchemaName(), util.getConnection());
assertTrue("Could not get phoenix columns.", columnListFromPhoenix.size() > 0);
- WriteWorkload loader = new WriteWorkload(util, parser, scenario);
+ WriteWorkload loader = new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO);
WorkloadExecutor executor = new WorkloadExecutor();
executor.add(loader);
executor.get();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/main/assembly/components-minimal.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/assembly/components-minimal.xml b/phoenix-pherf/src/main/assembly/components-minimal.xml
index 38c6929..9c9b9a1 100644
--- a/phoenix-pherf/src/main/assembly/components-minimal.xml
+++ b/phoenix-pherf/src/main/assembly/components-minimal.xml
@@ -23,7 +23,7 @@
<outputDirectory>/</outputDirectory>
<includes>
<include>org.apache.phoenix:phoenix-pherf</include>
- <include>org.jfree:jfreechart</include>
+ <include>org.jfree:*</include>
<include>com.googlecode.java-diff-utils:diffutils</include>
<include>org.apache.commons:commons-lang3</include>
<include>org.apache.commons:commons-math3</include>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
index 5a9f45f..0421b6f 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
@@ -19,6 +19,7 @@
package org.apache.phoenix.pherf;
import org.apache.commons.cli.*;
+import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
import org.apache.phoenix.pherf.configuration.XMLConfigParser;
import org.apache.phoenix.pherf.jmx.MonitorManager;
import org.apache.phoenix.pherf.schema.SchemaReader;
@@ -73,6 +74,8 @@ public class Pherf {
+ "See pherf.default.dataloader.threadpool in Pherf.properties.");
options.addOption("h", "help", false, "Get help on using this utility.");
options.addOption("d", "debug", false, "Put tool in debug mode");
+ options.addOption("stats", false,
+ "Update Phoenix Statistics after data is loaded with -l argument");
}
private final String zookeeper;
@@ -89,6 +92,7 @@ public class Pherf {
private final int rowCountOverride;
private final boolean listFiles;
private final boolean applySchema;
+ private final GeneratePhoenixStats generateStatistics;
public Pherf(String[] args) throws Exception {
CommandLineParser parser = new PosixParser();
@@ -126,6 +130,7 @@ public class Pherf {
command.hasOption("scenarioFile") ? command.getOptionValue("scenarioFile") : null;
schemaFile = command.hasOption("schemaFile") ? command.getOptionValue("schemaFile") : null;
rowCountOverride = Integer.parseInt(command.getOptionValue("rowCountOverride", "0"));
+ generateStatistics = command.hasOption("stats") ? GeneratePhoenixStats.YES : GeneratePhoenixStats.NO;
String
writerThreadPoolSize =
command.getOptionValue("writerThreadSize",
@@ -204,23 +209,11 @@ public class Pherf {
// Schema and Data Load
if (preLoadData) {
logger.info("\nStarting Data Load...");
- WriteWorkload workload = new WriteWorkload(parser);
+ WriteWorkload workload = new WriteWorkload(parser, generateStatistics);
workloadExecutor.add(workload);
// Wait for dataLoad to complete
workloadExecutor.get(workload);
-
- logger.info("\nGenerate query gold files after data load");
- QueryExecutor
- goldFileGenerator =
- new QueryExecutor(parser, phoenixUtil, workloadExecutor.getPool(),
- parser.getDataModels(), queryHint, true,
- PherfConstants.RunMode.FUNCTIONAL);
- workloadExecutor
- .add(goldFileGenerator);
-
- // Wait for dataLoad to complete
- workloadExecutor.get(goldFileGenerator);
} else {
logger.info(
"\nSKIPPED: Data Load and schema creation as -l argument not specified");
@@ -254,6 +247,10 @@ public class Pherf {
if (workloadExecutor != null) {
logger.info("Run completed. Shutting down thread pool.");
workloadExecutor.shutdown();
+ if (preLoadData) {
+ System.exit(0);
+ }
+
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
index e060e53..12580d4 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
@@ -62,6 +62,11 @@ public class PherfConstants {
public static final int MONITOR_FREQUENCY = 5000;
public static final String MONITOR_FILE_NAME = "STATS_MONITOR";
+ public static enum GeneratePhoenixStats {
+ YES,
+ NO
+ }
+
public static enum RunMode {
PERFORMANCE,
FUNCTIONAL
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 5f7d637..5b223b1 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -255,4 +255,15 @@ public class PhoenixUtil {
public static void setRowCountOverride(int rowCountOverride) {
PhoenixUtil.rowCountOverride = rowCountOverride;
}
+
+ /**
+ * Update Phoenix table stats
+ *
+ * @param tableName
+ * @throws Exception
+ */
+ public void updatePhoenixStats(String tableName) throws Exception {
+ logger.info("Updating stats for " + tableName);
+ executeStatement("UPDATE STATISTICS " + tableName);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java
index 624188c..4ab76c8 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/QueryExecutor.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.pherf.workload;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
import org.apache.phoenix.pherf.PherfConstants.RunMode;
import org.apache.phoenix.pherf.configuration.*;
import org.apache.phoenix.pherf.result.*;
@@ -142,7 +143,7 @@ public class QueryExecutor implements Workload {
for (int i = 0; i < writerThreadCount; i++) {
logger.debug("Inserting write workload ( " + i + " ) of ( "
+ writerThreadCount + " )");
- Workload writes = new WriteWorkload(PhoenixUtil.create(), parser);
+ Workload writes = new WriteWorkload(PhoenixUtil.create(), parser, GeneratePhoenixStats.NO);
pool.submit(writes.execute());
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index 305521b..b6686c6 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -19,6 +19,7 @@
package org.apache.phoenix.pherf.workload;
import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
import org.apache.phoenix.pherf.configuration.Column;
import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.configuration.WriteParams;
@@ -58,19 +59,24 @@ public class WriteWorkload implements Workload {
private final int threadPoolSize;
private final int batchSize;
+ private final GeneratePhoenixStats generateStatistics;
public WriteWorkload(XMLConfigParser parser) throws Exception {
- this(PhoenixUtil.create(), parser);
+ this(PhoenixUtil.create(), parser, GeneratePhoenixStats.NO);
+ }
+
+ public WriteWorkload(XMLConfigParser parser, GeneratePhoenixStats generateStatistics) throws Exception {
+ this(PhoenixUtil.create(), parser, generateStatistics);
}
- public WriteWorkload(PhoenixUtil util, XMLConfigParser parser) throws Exception {
- this(util, parser, null);
+ public WriteWorkload(PhoenixUtil util, XMLConfigParser parser, GeneratePhoenixStats generateStatistics) throws Exception {
+ this(util, parser, null, generateStatistics);
}
- public WriteWorkload(PhoenixUtil phoenixUtil, XMLConfigParser parser, Scenario scenario)
+ public WriteWorkload(PhoenixUtil phoenixUtil, XMLConfigParser parser, Scenario scenario, GeneratePhoenixStats generateStatistics)
throws Exception {
this(phoenixUtil, PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES),
- parser, scenario);
+ parser, scenario, generateStatistics);
}
/**
@@ -87,11 +93,12 @@ public class WriteWorkload implements Workload {
* @throws Exception
*/
public WriteWorkload(PhoenixUtil phoenixUtil, Properties properties, XMLConfigParser parser,
- Scenario scenario) throws Exception {
+ Scenario scenario, GeneratePhoenixStats generateStatistics) throws Exception {
this.pUtil = phoenixUtil;
this.parser = parser;
this.rulesApplier = new RulesApplier(parser);
this.resultUtil = new ResultUtil();
+ this.generateStatistics = generateStatistics;
// Overwrite defaults properties with those given in the configuration. This indicates the
// scenario is a R/W mixed workload.
@@ -156,9 +163,15 @@ public class WriteWorkload implements Workload {
List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario);
waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches);
-
- // always update stats for Phoenix base tables
- updatePhoenixStats(scenario.getTableName());
+
+ // Update Phoenix Statistics
+ if (this.generateStatistics == GeneratePhoenixStats.YES) {
+ logger.info("Updating Phoenix table statistics...");
+ pUtil.updatePhoenixStats(scenario.getTableName());
+ logger.info("Stats update done!");
+ } else {
+ logger.info("Phoenix table stats update not requested.");
+ }
}
private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario)
@@ -208,18 +221,6 @@ public class WriteWorkload implements Workload {
.add(scenario.getTableName(), sumRows, (int) (System.currentTimeMillis() - start));
}
- /**
- * TODO Move this method to PhoenixUtil
- * Update Phoenix table stats
- *
- * @param tableName
- * @throws Exception
- */
- public void updatePhoenixStats(String tableName) throws Exception {
- logger.info("Updating stats for " + tableName);
- pUtil.executeStatement("UPDATE STATISTICS " + tableName);
- }
-
public Future<Info> upsertData(final Scenario scenario, final List<Column> columns,
final String tableName, final int rowCount,
final DataLoadThreadTime dataLoadThreadTime) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/060a00ca/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
index 94c8171..731a57a 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
@@ -34,7 +34,7 @@ public class PherfTest {
@Test
public void testUnknownOption() {
- String[] args = {"-drop", "all", "-l", "-q", "-m","-bsOption"};
+ String[] args = {"-drop", "all", "-q", "-m","-bsOption"};
// Makes sure that System.exit(1) is called. Release is a required param.
exit.expectSystemExitWithStatus(1);