You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2021/09/20 12:30:55 UTC
[phoenix] branch 4.16 updated: PHOENIX-6553 Sync phoenix-pherf in
4.16 to 4.x
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch 4.16
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.16 by this push:
new 64e216e PHOENIX-6553 Sync phoenix-pherf in 4.16 to 4.x
64e216e is described below
commit 64e216eb265c387663f9c8de1bafcb773d952859
Author: Istvan Toth <st...@apache.org>
AuthorDate: Mon Sep 20 09:03:49 2021 +0200
PHOENIX-6553 Sync phoenix-pherf in 4.16 to 4.x
---
phoenix-pherf/pom.xml | 30 +-
.../java/org/apache/phoenix/pherf/PherfMainIT.java | 6 +-
.../org/apache/phoenix/pherf/ResultBaseTestIT.java | 8 +-
.../org/apache/phoenix/pherf/SchemaReaderIT.java | 2 +-
.../pherf/workload/mt/MultiTenantTestUtils.java | 330 +++++++++++++++++
.../mt/TenantTableOperationWorkloadIT.java | 149 ++++++++
.../workload/mt/TenantViewOperationWorkloadIT.java | 157 ++++++++
.../datamodel/create_prod_test_unsalted.sql | 33 ++
phoenix-pherf/src/it/resources/hbase-site.xml | 25 ++
phoenix-pherf/src/it/resources/pherf.properties | 42 +++
.../scenario/prod_test_unsalted_scenario.xml | 410 +++++++++++++++++++++
.../main/java/org/apache/phoenix/pherf/Pherf.java | 69 +++-
.../org/apache/phoenix/pherf/PherfConstants.java | 6 +
.../pherf/configuration/DataTypeMapping.java | 2 +
.../apache/phoenix/pherf/configuration/Ddl.java | 14 +-
.../phoenix/pherf/configuration/IdleTime.java | 47 +++
.../phoenix/pherf/configuration/LoadProfile.java | 121 ++++++
.../pherf/configuration/OperationGroup.java | 44 +++
.../apache/phoenix/pherf/configuration/Query.java | 54 +--
.../phoenix/pherf/configuration/Scenario.java | 65 +++-
.../phoenix/pherf/configuration/TenantGroup.java | 62 ++++
.../apache/phoenix/pherf/configuration/Upsert.java | 129 +++++++
.../phoenix/pherf/configuration/UserDefined.java | 55 +++
.../apache/phoenix/pherf/jmx/MonitorManager.java | 4 +-
.../org/apache/phoenix/pherf/result/Result.java | 2 +-
.../apache/phoenix/pherf/result/ResultManager.java | 3 +-
.../pherf/rules/RuleBasedDataGenerator.java | 2 +-
.../apache/phoenix/pherf/rules/RulesApplier.java | 136 ++++---
.../pherf/rules/SequentialDateDataGenerator.java | 68 ++++
.../pherf/rules/SequentialListDataGenerator.java | 66 ++++
.../rules/SequentialVarcharDataGenerator.java | 75 ++++
.../org/apache/phoenix/pherf/util/PhoenixUtil.java | 196 +++++++++-
.../apache/phoenix/pherf/util/ResourceList.java | 5 +-
.../pherf/workload/MultiThreadedRunner.java | 2 +-
.../phoenix/pherf/workload/WriteWorkload.java | 131 +------
.../pherf/workload/mt/MultiTenantWorkload.java | 81 ++++
.../mt/generators/BaseLoadEventGenerator.java | 216 +++++++++++
.../workload/mt/generators/LoadEventGenerator.java | 62 ++++
.../mt/generators/LoadEventGeneratorFactory.java | 43 +++
.../generators/SequentialLoadEventGenerator.java | 187 ++++++++++
.../TenantLoadEventGeneratorFactory.java | 70 ++++
.../mt/generators/TenantOperationInfo.java | 70 ++++
.../UniformDistributionLoadEventGenerator.java | 109 ++++++
.../WeightedRandomLoadEventGenerator.java | 186 ++++++++++
.../workload/mt/handlers/PherfWorkHandler.java | 29 ++
.../mt/handlers/RendezvousingWorkHandler.java | 103 ++++++
.../mt/handlers/TenantOperationWorkHandler.java | 87 +++++
.../mt/operations/BaseOperationSupplier.java | 48 +++
.../workload/mt/operations/IdleTimeOperation.java | 29 ++
.../mt/operations/IdleTimeOperationSupplier.java | 78 ++++
.../pherf/workload/mt/operations/Operation.java | 31 ++
.../workload/mt/operations/OperationStats.java | 108 ++++++
.../mt/operations/PreScenarioOperation.java | 31 ++
.../operations/PreScenarioOperationSupplier.java | 89 +++++
.../workload/mt/operations/QueryOperation.java | 29 ++
.../mt/operations/QueryOperationSupplier.java | 98 +++++
.../mt/operations/TenantOperationFactory.java | 365 ++++++++++++++++++
.../workload/mt/operations/UpsertOperation.java | 29 ++
.../mt/operations/UpsertOperationSupplier.java | 165 +++++++++
.../mt/operations/UserDefinedOperation.java | 29 ++
.../operations/UserDefinedOperationSupplier.java | 51 +++
.../phoenix/pherf/ConfigurationParserTest.java | 113 +++++-
.../java/org/apache/phoenix/pherf/PherfTest.java | 2 +-
.../org/apache/phoenix/pherf/ResultBaseTest.java | 6 +-
.../apache/phoenix/pherf/RuleGeneratorTest.java | 5 +-
.../rules/SequentialDateDataGeneratorTest.java | 78 ++++
.../rules/SequentialListDataGeneratorTest.java | 86 +++++
.../rules/SequentialVarcharDataGeneratorTest.java | 68 ++++
.../mt/SequentialLoadEventGeneratorTest.java | 146 ++++++++
.../workload/mt/TenantOperationFactoryTest.java | 128 +++++++
.../UniformDistributionLoadEventGeneratorTest.java | 136 +++++++
.../mt/WeightedRandomLoadEventGeneratorTest.java | 219 +++++++++++
...st_schema.sql => test_mt_schema_base_table.sql} | 20 +-
.../{test_schema.sql => test_mt_schema_view1.sql} | 26 +-
.../{test_schema.sql => test_mt_schema_view2.sql} | 26 +-
.../src/test/resources/datamodel/test_schema.sql | 4 +-
...{test_schema.sql => test_tbl_schema_simple.sql} | 32 +-
.../scenario/malicious_scenario_with_dtd.xml | 2 +-
.../src/test/resources/scenario/test_evt_gen1.xml | 70 ++++
.../src/test/resources/scenario/test_evt_gen2.xml | 82 +++++
.../src/test/resources/scenario/test_evt_gen3.xml | 78 ++++
.../src/test/resources/scenario/test_evt_gen4.xml | 78 ++++
.../scenario/test_mt_workload_template.xml | 226 ++++++++++++
.../src/test/resources/scenario/test_scenario.xml | 29 +-
.../scenario/test_tbl_workload_template.xml | 169 +++++++++
...rio.xml => test_workload_with_load_profile.xml} | 295 +++++++++------
86 files changed, 6647 insertions(+), 450 deletions(-)
diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml
index 2b5d781..86968e3 100644
--- a/phoenix-pherf/pom.xml
+++ b/phoenix-pherf/pom.xml
@@ -63,15 +63,24 @@
<version>3.3.2</version>
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.3</version>
</dependency>
- <dependency>
- <groupId>org.apache.phoenix.thirdparty</groupId>
- <artifactId>phoenix-shaded-commons-cli</artifactId>
- </dependency>
-
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix.thirdparty</groupId>
+ <artifactId>phoenix-shaded-commons-cli</artifactId>
+ </dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>junit</groupId>
@@ -127,7 +136,7 @@
<build>
<resources>
<resource>
- <directory>src/main/resources</directory>
+ <directory>src/it/resources</directory>
</resource>
<resource>
<directory>config</directory>
@@ -142,6 +151,9 @@
<directory>src/test/resources</directory>
</testResource>
<testResource>
+ <directory>src/it/resources</directory>
+ </testResource>
+ <testResource>
<directory>${project.basedir}/config</directory>
</testResource>
</testResources>
@@ -212,7 +224,7 @@
<artifactSet>
<includes>
<include>org.apache.phoenix:phoenix-pherf</include>
- <include>com.google.guava:guava</include>
+ <include>org.apache.phoenix.thirdparty:phoenix-shaded-guava</include>
<include>com.googlecode.java-diff-utils:diffutils</include>
<include>org.apache.commons:commons-lang3</include>
<include>org.apache.commons:commons-math3</include>
@@ -220,6 +232,10 @@
<include>joda-time:joda-time</include>
<include>org.apache.commons:commons-csv</include>
<include>commons-lang:commons-lang</include>
+ <include>commons-io:commons-io</include>
+ <include>com.google.code.gson:gson</include>
+ <include>com.lmax:disruptor</include>
+ <include>commons-io:commons-io</include>
</includes>
</artifactSet>
<filters>
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/PherfMainIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/PherfMainIT.java
index c1a7b66..b9cfbcf 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/PherfMainIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/PherfMainIT.java
@@ -23,6 +23,7 @@ import org.apache.phoenix.pherf.result.Result;
import org.apache.phoenix.pherf.result.ResultValue;
import org.apache.phoenix.pherf.result.file.ResultFileDetails;
import org.apache.phoenix.pherf.result.impl.CSVFileResultHandler;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.ExpectedSystemExit;
@@ -53,9 +54,8 @@ public class PherfMainIT extends ResultBaseTestIT {
@Test
public void testPherfMain() throws Exception {
String[] args = { "-q", "-l",
- "--schemaFile", ".*create_prod_test_unsalted.sql",
- "--scenarioFile", ".*prod_test_unsalted_scenario.xml",
- "-m", "--monitorFrequency", "10" };
+ "-schemaFile", ".*create_prod_test_unsalted.sql",
+ "-scenarioFile", ".*prod_test_unsalted_scenario.xml"};
Pherf pherf = new Pherf(args);
pherf.run();
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
index fe1f2ea..f3d7cba 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/ResultBaseTestIT.java
@@ -25,11 +25,11 @@ import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.pherf.configuration.XMLConfigParser;
import org.apache.phoenix.pherf.result.ResultUtil;
import org.apache.phoenix.pherf.schema.SchemaReader;
import org.apache.phoenix.pherf.util.PhoenixUtil;
-import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.After;
import org.junit.AfterClass;
@@ -37,9 +37,9 @@ import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
@Category(NeedsOwnMiniClusterTest.class)
-public class ResultBaseTestIT extends BaseTest {
- protected static final String matcherScenario = ".*scenario/.*test.*xml";
- protected static final String matcherSchema = ".*datamodel/.*test.*sql";
+public class ResultBaseTestIT extends ParallelStatsDisabledIT {
+ protected static final String matcherScenario = ".*scenario/.*test_scenario.xml";
+ protected static final String matcherSchema = ".*datamodel/.*test_schema.sql";
protected static PhoenixUtil util = PhoenixUtil.create(true);
protected static Properties properties;
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java
index 901c92f..c5a93fe 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java
@@ -78,7 +78,7 @@ public class SchemaReaderIT extends BaseTest {
private void assertApplySchemaTest() {
try {
util.setZookeeper("localhost");
- SchemaReader reader = new SchemaReader(util, ".*datamodel/.*test.*sql");
+ SchemaReader reader = new SchemaReader(util, ".*datamodel/.*test_schema.*sql");
List<Path> resources = new ArrayList<>(reader.getResourceList());
assertTrue("Could not pull list of schema files.", resources.size() > 0);
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/MultiTenantTestUtils.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/MultiTenantTestUtils.java
new file mode 100644
index 0000000..c99f136
--- /dev/null
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/MultiTenantTestUtils.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import com.lmax.disruptor.LifecycleAware;
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.XMLConfigParserTest;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.result.ResultValue;
+import org.apache.phoenix.pherf.schema.SchemaReader;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.Workload;
+import org.apache.phoenix.pherf.workload.WorkloadExecutor;
+import org.apache.phoenix.pherf.workload.mt.generators.LoadEventGenerator;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantLoadEventGeneratorFactory;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.operations.IdleTimeOperationSupplier;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+import org.apache.phoenix.pherf.workload.mt.operations.OperationStats;
+import org.apache.phoenix.pherf.workload.mt.operations.QueryOperationSupplier;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+import org.apache.phoenix.pherf.workload.mt.operations.UpsertOperationSupplier;
+import org.apache.phoenix.pherf.workload.mt.operations.UserDefinedOperationSupplier;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.phoenix.pherf.workload.mt.generators.BaseLoadEventGenerator.TenantOperationEvent;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MultiTenantTestUtils {
+ private static final Logger LOGGER = LoggerFactory.getLogger(MultiTenantTestUtils.class);
+ enum TestOperationGroup {
+ upsertOp, queryOp1, queryOp2, idleOp, udfOp
+ }
+
+ public static class TestConfigAndExpectations {
+ List<TenantGroup> tenantGroups;
+ int expectedTenantGroups;
+ int expectedOpGroups;
+ }
+
+ public SchemaReader applySchema(PhoenixUtil util, String matcherSchema) throws Exception {
+ PherfConstants constants = PherfConstants.create();
+
+ PhoenixUtil.setZookeeper("localhost");
+ SchemaReader reader = new SchemaReader(util, matcherSchema);
+ reader.applySchema();
+ List<Path> resources = new ArrayList<>(reader.getResourceList());
+
+ assertTrue("Could not pull list of schema files.", resources.size() > 0);
+ assertNotNull("Could not read schema file.", reader.resourceToString(resources.get(0)));
+ return reader;
+ }
+
+ public DataModel readTestDataModel(String resourceName) throws Exception {
+ URL scenarioUrl = XMLConfigParserTest.class.getResource(resourceName);
+ assertNotNull(scenarioUrl);
+ Path p = Paths.get(scenarioUrl.toURI());
+ return XMLConfigParser.readDataModel(p);
+ }
+
+ public void testWorkloadWithCountingHandlers(Properties properties, DataModel model,
+ String scenarioName,
+ int numHandlers,
+ int expectedTenantGroups, int expectedOpGroups) throws Exception {
+
+ int totalOperations = 500;
+ int perHandlerCount = 50;
+
+ List<Workload> workloads = new ArrayList<>();
+ WorkloadExecutor workloadExecutor = new WorkloadExecutor(properties, workloads, false);
+ try {
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ for (Scenario scenario : model.getScenarios()) {
+ if (scenarioName != null && !scenarioName.isEmpty()
+ && scenario.getName().compareTo(scenarioName) != 0) {
+ continue;
+ }
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+
+ // Set the total number of operations for this load profile
+ loadProfile.setNumOperations(totalOperations);
+ TenantOperationFactory opFactory = new TenantOperationFactory(pUtil, model, scenario);
+ assertEquals("tenant group size is not as expected: ", expectedTenantGroups,
+ loadProfile.getTenantDistribution().size());
+
+ assertEquals("operation group size from the factory is not as expected: ",
+ expectedOpGroups, opFactory.getOperations().size());
+
+ // populate the handlers and countdown latches.
+ List<PherfWorkHandler> workers = Lists.newArrayList();
+ Map<String, CountDownLatch> latches = Maps.newConcurrentMap();
+ for (int i = 0; i < numHandlers; i++) {
+ String handlerId = String
+ .format("%s.%d", InetAddress.getLocalHost().getHostName(), i);
+ workers.add(new EventCountingWorkHandler(opFactory, handlerId, latches));
+ latches.put(handlerId, new CountDownLatch(perHandlerCount));
+ }
+
+ // submit the workload
+ Workload workload = new MultiTenantWorkload(pUtil, model, scenario, workers,
+ properties);
+ workloads.add(workload);
+ workloadExecutor.add(workload);
+ // Just make sure there are no exceptions
+ workloadExecutor.get();
+
+ // Wait for the handlers to count down
+ for (Map.Entry<String, CountDownLatch> latch : latches.entrySet()) {
+ assertTrue(latch.getValue().await(60, TimeUnit.SECONDS));
+ }
+ }
+ } finally {
+ if (!workloads.isEmpty()) {
+ for (Workload workload : workloads) {
+ workload.complete();
+ }
+ }
+ workloadExecutor.complete();
+ workloadExecutor.shutdown();
+ }
+
+ }
+
+ public void testWorkloadWithHandlers(Properties properties, DataModel model,
+ String scenarioName,
+ int numHandlers,
+ int expectedTenantGroups, int expectedOpGroups) throws Exception {
+
+ int totalOperations = 500;
+
+ List<Workload> workloads = new ArrayList<>();
+ WorkloadExecutor workloadExecutor = new WorkloadExecutor(properties, workloads, false);
+ try {
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ for (Scenario scenario : model.getScenarios()) {
+ if (scenarioName != null && !scenarioName.isEmpty()
+ && scenario.getName().compareTo(scenarioName) != 0) {
+ continue;
+ }
+
+ Map<String, String> scenarioProperties = Maps.newHashMap();
+ scenarioProperties.put("pherf.mt.handlers_per_scenario", String.valueOf(numHandlers));
+ scenario.setPhoenixProperties(scenarioProperties);
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+
+ // Set the total number of operations for this load profile
+ loadProfile.setNumOperations(totalOperations);
+ TenantOperationFactory opFactory = new TenantOperationFactory(pUtil, model, scenario);
+ assertEquals("tenant group size is not as expected: ", expectedTenantGroups,
+ loadProfile.getTenantDistribution().size());
+
+ assertEquals("operation group size from the factory is not as expected: ",
+ expectedOpGroups, opFactory.getOperations().size());
+ // submit the workload
+ Workload workload = new MultiTenantWorkload(pUtil, model, scenario,
+ properties);
+ workloads.add(workload);
+ workloadExecutor.add(workload);
+ // Just make sure there are no exceptions
+ workloadExecutor.get();
+
+ }
+ } finally {
+ if (!workloads.isEmpty()) {
+ for (Workload workload : workloads) {
+ workload.complete();
+ }
+ }
+ workloadExecutor.complete();
+ workloadExecutor.shutdown();
+ }
+
+ }
+
+ public void testVariousOperations(Properties properties, DataModel model, String scenarioName,
+ int expectedTenantGroups, int expectedOpGroups) throws Exception {
+
+ int numRuns = 10;
+ int numOperations = 10;
+
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ for (Scenario scenario : model.getScenarios()) {
+ if (scenarioName != null && !scenarioName.isEmpty()
+ && scenario.getName().compareTo(scenarioName) != 0) {
+ continue;
+ }
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+ assertEquals("tenant group size is not as expected: ", expectedTenantGroups,
+ loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ", expectedOpGroups,
+ loadProfile.getOpDistribution().size());
+
+ TenantLoadEventGeneratorFactory eventGenFactory = new TenantLoadEventGeneratorFactory();
+ LoadEventGenerator evtGen = eventGenFactory.newLoadEventGenerator(
+ pUtil, model, scenario, properties);
+ TenantOperationFactory opFactory = evtGen.getOperationFactory();
+
+ assertEquals("operation group size from the factory is not as expected: ",
+ expectedOpGroups, opFactory.getOperations().size());
+
+ int numRowsInserted = 0;
+ for (int i = 0; i < numRuns; i++) {
+ int ops = numOperations;
+ loadProfile.setNumOperations(ops);
+ while (ops-- > 0) {
+ TenantOperationInfo info = evtGen.next();
+ opFactory.initializeTenant(info);
+ Supplier<Function<TenantOperationInfo, OperationStats>> opSupplier = opFactory
+ .getOperationSupplier(info);
+ OperationStats stats = opSupplier.get().apply(info);
+ LOGGER.info(PhoenixUtil.getGSON().toJson(stats));
+ if (info.getOperation().getType() == Operation.OperationType.PRE_RUN) continue;
+ assertTrue(stats.getStatus() != -1);
+ switch (TestOperationGroup
+ .valueOf(info.getOperationGroupId())) {
+ case upsertOp:
+ assertTrue(opSupplier.getClass()
+ .isAssignableFrom(UpsertOperationSupplier.class));
+ numRowsInserted += stats.getRowCount();
+ break;
+ case queryOp1:
+ case queryOp2:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(QueryOperationSupplier.class));
+
+ // expected row count >= 0
+ // Since the same view/table is being used by many tests.
+ // Keeping query return values would require lot of housekeeping
+ assertTrue(stats.getRowCount() >= 0);
+ break;
+ case idleOp:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(IdleTimeOperationSupplier.class));
+ assertEquals(0, stats.getRowCount());
+ // expected think time (no-op) to be ~50ms
+ assertTrue(25 < stats.getDurationInMs() && stats.getDurationInMs() < 75);
+ break;
+ case udfOp:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(UserDefinedOperationSupplier.class));
+ assertEquals(0, stats.getRowCount());
+ break;
+ default:
+ Assert.fail();
+ }
+ }
+ }
+ }
+ }
+
+ private static class EventCountingWorkHandler
+ implements PherfWorkHandler<TenantOperationEvent>, LifecycleAware {
+ private final String handlerId;
+ private final TenantOperationFactory tenantOperationFactory;
+ private final Map<String, CountDownLatch> latches;
+
+ public EventCountingWorkHandler(TenantOperationFactory tenantOperationFactory,
+ String handlerId, Map<String, CountDownLatch> latches) {
+ this.handlerId = handlerId;
+ this.tenantOperationFactory = tenantOperationFactory;
+ this.latches = latches;
+ }
+
+ @Override public void onStart() {
+ }
+
+ @Override public void onShutdown() {
+ }
+
+ @Override public void onEvent(TenantOperationEvent event)
+ throws Exception {
+ TenantOperationInfo input = event.getTenantOperationInfo();
+ Supplier<Function<TenantOperationInfo, OperationStats>>
+ opSupplier
+ = tenantOperationFactory.getOperationSupplier(input);
+ OperationStats stats = opSupplier.get().apply(input);
+ LOGGER.info(PhoenixUtil.getGSON().toJson(stats));
+ assertEquals(0, stats.getStatus());
+ latches.get(handlerId).countDown();
+ }
+
+ @Override public List<ResultValue<OperationStats>> getResults() {
+ return null;
+ }
+ }
+
+}
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/TenantTableOperationWorkloadIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/TenantTableOperationWorkloadIT.java
new file mode 100644
index 0000000..b318ef0
--- /dev/null
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/TenantTableOperationWorkloadIT.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import com.google.common.collect.Lists;
+import com.lmax.disruptor.WorkHandler;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantLoadEventGeneratorFactory.GeneratorType;
+import org.apache.phoenix.pherf.workload.mt.MultiTenantTestUtils.TestConfigAndExpectations;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Tests focused on tenant tablee operations and their validations
+ * Tests focused on tenant operation workloads {@link MultiTenantWorkload}
+ * and workload handlers {@link WorkHandler}
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+@RunWith(Parameterized.class)
+public class TenantTableOperationWorkloadIT extends ParallelStatsDisabledIT {
+
+ private final MultiTenantTestUtils multiTenantTestUtils = new MultiTenantTestUtils();
+ private final Properties properties = PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+ private final PhoenixUtil util = PhoenixUtil.create(true);
+ private GeneratorType generatorType;
+
+ public TenantTableOperationWorkloadIT(String generatorType) throws Exception {
+ this.generatorType = GeneratorType.valueOf(generatorType);
+ }
+
+ @Parameterized.Parameters( name = "generator_type={0}" )
+ public static synchronized Collection<Object[]> data() {
+ List<Object[]> testCases = Lists.newArrayList();
+ testCases.add(new Object[] { "WEIGHTED" });
+ testCases.add(new Object[] { "UNIFORM" });
+ testCases.add(new Object[] { "SEQUENTIAL" });
+ return testCases;
+ }
+
+ @Before
+ public void setup() throws Exception {
+ multiTenantTestUtils.applySchema(util, ".*datamodel/.*test_tbl_schema_simple.sql");
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ util.deleteTables("PHERF.*MULTI_TENANT_TABLE");
+ }
+
+ @Test
+ public void testVariousOperations() throws Exception {
+ DataModel model = multiTenantTestUtils.readTestDataModel(
+ "/scenario/test_tbl_workload_template.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ TestConfigAndExpectations settings = getTestConfigAndExpectations(scenario, generatorType);
+ scenario.setGeneratorName(generatorType.name());
+ scenario.getLoadProfile().setTenantDistribution(settings.tenantGroups);
+ multiTenantTestUtils.testVariousOperations(properties, model, scenario.getName(),
+ settings.expectedTenantGroups, settings.expectedOpGroups);
+ }
+ }
+
+ @Test
+ public void testWorkloadWithOneHandler() throws Exception {
+ int numHandlers = 1;
+
+ DataModel model = multiTenantTestUtils.readTestDataModel(
+ "/scenario/test_tbl_workload_template.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ TestConfigAndExpectations settings = getTestConfigAndExpectations(scenario, generatorType);
+ scenario.setGeneratorName(generatorType.name());
+ scenario.getLoadProfile().setTenantDistribution(settings.tenantGroups);
+ multiTenantTestUtils.testWorkloadWithHandlers(properties, model, scenario.getName(),
+ numHandlers, settings.expectedTenantGroups, settings.expectedOpGroups);
+ }
+ }
+
+ @Test
+ public void testWorkloadWithManyHandlers() throws Exception {
+ int numHandlers = 5;
+ DataModel model = multiTenantTestUtils.readTestDataModel(
+ "/scenario/test_tbl_workload_template.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ TestConfigAndExpectations settings = getTestConfigAndExpectations(scenario, generatorType);
+ scenario.setGeneratorName(generatorType.name());
+ scenario.getLoadProfile().setTenantDistribution(settings.tenantGroups);
+ multiTenantTestUtils.testWorkloadWithHandlers(properties, model, scenario.getName(),
+ numHandlers, settings.expectedTenantGroups, settings.expectedOpGroups);
+ }
+ }
+
+ private TestConfigAndExpectations getTestConfigAndExpectations(
+ Scenario scenario, GeneratorType generatorType) {
+
+ TestConfigAndExpectations
+ settings = new TestConfigAndExpectations();
+
+ switch (generatorType) {
+ case WEIGHTED:
+ settings.tenantGroups = scenario.getLoadProfile().getTenantDistribution();
+ settings.expectedOpGroups = scenario.getLoadProfile().getOpDistribution().size();
+ settings.expectedTenantGroups = scenario.getLoadProfile().getTenantDistribution().size();
+ default:
+ List<TenantGroup> tenantGroups = new ArrayList<>();
+ TenantGroup tg1 = new TenantGroup();
+ tg1.setId("tg1");
+ tg1.setNumTenants(10);
+ tg1.setWeight(100);
+ tenantGroups.add(tg1);
+ settings.tenantGroups = tenantGroups;
+ settings.expectedTenantGroups = 1;
+ settings.expectedOpGroups = scenario.getLoadProfile().getOpDistribution().size();;
+ break;
+ }
+ return settings;
+ }
+}
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/TenantViewOperationWorkloadIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/TenantViewOperationWorkloadIT.java
new file mode 100644
index 0000000..18e72c0
--- /dev/null
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/workload/mt/TenantViewOperationWorkloadIT.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import com.google.common.collect.Lists;
+import com.lmax.disruptor.WorkHandler;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantLoadEventGeneratorFactory.GeneratorType;
+import org.apache.phoenix.pherf.workload.mt.MultiTenantTestUtils.TestConfigAndExpectations;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Tests focused on tenant view operations and their validations
+ * Tests focused on tenant operation workloads {@link MultiTenantWorkload}
+ * and workload handlers {@link WorkHandler}
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+@RunWith(Parameterized.class)
+public class TenantViewOperationWorkloadIT extends ParallelStatsDisabledIT {
+ private final MultiTenantTestUtils multiTenantTestUtils = new MultiTenantTestUtils();
+ private final Properties properties = PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+ private final PhoenixUtil util = PhoenixUtil.create(true);
+ private final RegionCoprocessorEnvironment taskRegionEnvironment;
+ private GeneratorType generatorType;
+
+ public TenantViewOperationWorkloadIT(String generatorType) throws Exception {
+ this.generatorType = GeneratorType.valueOf(generatorType);
+ taskRegionEnvironment =
+ (RegionCoprocessorEnvironment)getUtility()
+ .getRSForFirstRegionInTable(
+ PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME)
+ .getOnlineRegions(PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME)
+ .get(0).getCoprocessorHost()
+ .findCoprocessorEnvironment(TaskRegionObserver.class.getName());
+ }
+
+ @Parameterized.Parameters( name = "generator_type={0}" )
+ public static synchronized Collection<Object[]> data() {
+ List<Object[]> testCases = Lists.newArrayList();
+ testCases.add(new Object[] { "WEIGHTED" });
+ testCases.add(new Object[] { "UNIFORM" });
+ testCases.add(new Object[] { "SEQUENTIAL" });return testCases;
+ }
+
+ @Before
+ public void setup() throws Exception {
+ multiTenantTestUtils.applySchema(util,".*datamodel/.*test_mt.*.sql");
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ util.deleteTables("PHERF.*BASE_TABLE");
+ if (taskRegionEnvironment != null) {
+ util.dropChildView(taskRegionEnvironment, 2);
+ }
+ }
+
+ @Test public void testVariousOperations() throws Exception {
+
+ DataModel model = multiTenantTestUtils.readTestDataModel(
+ "/scenario/test_mt_workload_template.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ TestConfigAndExpectations settings = getTestConfigAndExpectations(scenario, generatorType);
+ scenario.setGeneratorName(generatorType.name());
+ scenario.getLoadProfile().setTenantDistribution(settings.tenantGroups);
+ multiTenantTestUtils.testVariousOperations(properties, model, scenario.getName(),
+ settings.expectedTenantGroups, settings.expectedOpGroups);
+ }
+ }
+
+ @Test public void testWorkloadWithOneHandler() throws Exception {
+ int numHandlers = 1;
+
+ DataModel model = multiTenantTestUtils.readTestDataModel(
+ "/scenario/test_mt_workload_template.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ TestConfigAndExpectations settings = getTestConfigAndExpectations(scenario, generatorType);
+ scenario.setGeneratorName(generatorType.name());
+ scenario.getLoadProfile().setTenantDistribution(settings.tenantGroups);
+ multiTenantTestUtils.testWorkloadWithHandlers(properties, model, scenario.getName(),
+ numHandlers, settings.expectedTenantGroups, settings.expectedOpGroups);
+ }
+ }
+
+ @Test public void testWorkloadWithManyHandlers() throws Exception {
+ int numHandlers = 5;
+ DataModel model = multiTenantTestUtils.readTestDataModel(
+ "/scenario/test_mt_workload_template.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ TestConfigAndExpectations settings = getTestConfigAndExpectations(scenario, generatorType);
+ scenario.setGeneratorName(generatorType.name());
+ scenario.getLoadProfile().setTenantDistribution(settings.tenantGroups);
+ multiTenantTestUtils.testWorkloadWithHandlers(properties, model, scenario.getName(),
+ numHandlers, settings.expectedTenantGroups, settings.expectedOpGroups);
+ }
+ }
+
+ private TestConfigAndExpectations getTestConfigAndExpectations(Scenario scenario, GeneratorType generatorType) {
+ TestConfigAndExpectations settings = new TestConfigAndExpectations();
+
+ switch (generatorType) {
+ case WEIGHTED:
+ settings.tenantGroups = scenario.getLoadProfile().getTenantDistribution();
+ settings.expectedOpGroups = scenario.getLoadProfile().getOpDistribution().size();
+ settings.expectedTenantGroups = scenario.getLoadProfile().getTenantDistribution().size();
+ default:
+ List<TenantGroup> tenantGroups = new ArrayList<>();
+ TenantGroup tg1 = new TenantGroup();
+ tg1.setId("tg1");
+ tg1.setNumTenants(10);
+ tg1.setWeight(100);
+ tenantGroups.add(tg1);
+ settings.tenantGroups = tenantGroups;
+ settings.expectedTenantGroups = 1;
+ settings.expectedOpGroups = scenario.getLoadProfile().getOpDistribution().size();;
+ break;
+ }
+ return settings;
+ }
+}
diff --git a/phoenix-pherf/src/it/resources/datamodel/create_prod_test_unsalted.sql b/phoenix-pherf/src/it/resources/datamodel/create_prod_test_unsalted.sql
new file mode 100644
index 0000000..dd1e2d8
--- /dev/null
+++ b/phoenix-pherf/src/it/resources/datamodel/create_prod_test_unsalted.sql
@@ -0,0 +1,33 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+CREATE TABLE IF NOT EXISTS PHERF.PHERF_PROD_TEST_UNSALTED (
+ TENANT_ID CHAR(15) NOT NULL,
+ CREATED_DATE DATE NOT NULL,
+ FIELD VARCHAR,
+ DATA_TYPE VARCHAR,
+ OLDVAL_STRING VARCHAR,
+ NEWVAL_STRING VARCHAR,
+ DIVISION INTEGER,
+ CONNECTION_ID VARCHAR
+ CONSTRAINT PK PRIMARY KEY
+ (
+ TENANT_ID,
+ CREATED_DATE DESC
+ )
+) VERSIONS=1,MULTI_TENANT=true
diff --git a/phoenix-pherf/src/it/resources/hbase-site.xml b/phoenix-pherf/src/it/resources/hbase-site.xml
new file mode 100644
index 0000000..8b7d0db
--- /dev/null
+++ b/phoenix-pherf/src/it/resources/hbase-site.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>phoenix.query.threadPoolSize</name>
+ <value>128</value>
+ </property>
+</configuration>
diff --git a/phoenix-pherf/src/it/resources/pherf.properties b/phoenix-pherf/src/it/resources/pherf.properties
new file mode 100644
index 0000000..b5a5e62
--- /dev/null
+++ b/phoenix-pherf/src/it/resources/pherf.properties
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# General purpose thread pool size for Pherf. It's used for things like monitor threads.This should remain small
+# to limit the amount of background tasks sucking up resources away from tests.
+pherf.default.threadpool=10
+
+# Interval in Ms that the JMX monitors will take a snapshot and dump to log
+pherf.default.monitorFrequency=30000
+
+# Default value to display log line after every 'N' row load
+pherf.default.log_per_nrows=1000000
+
+# Default number of writers to use when loading data
+# 0 - Set the number of writers to use all available cores
+# 1-N - ANy integer value for the number of threads to use
+pherf.default.dataloader.threadpool=0
+
+# When upserting, this is the max # of rows that will be inserted in a single commit
+pherf.default.dataloader.batchsize=1000
+
+# Directory where results from a scenario run will be written
+pherf.default.results.dir=/tmp/RESULTS
+
+# Google chart summary html file
+pherf.default.summary.file=/tmp/RESULTS/summary.html
+
+# Threshold for comparator to fail. ex. 0.5 equates to 50%
+pherf.default.comparison.threshold=0.45
diff --git a/phoenix-pherf/src/it/resources/scenario/prod_test_unsalted_scenario.xml b/phoenix-pherf/src/it/resources/scenario/prod_test_unsalted_scenario.xml
new file mode 100644
index 0000000..0c1de9e
--- /dev/null
+++ b/phoenix-pherf/src/it/resources/scenario/prod_test_unsalted_scenario.xml
@@ -0,0 +1,410 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<datamodel name="PROD_TEST_UNSALTED">
+ <datamapping>
+ <column>
+ <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
+ <type>VARCHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_VARCHAR</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_CHAR</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <!--SEQUENTIAL is unsupported for DATE -->
+ <dataSequence>RANDOM</dataSequence>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <minValue>1975</minValue>
+ <maxValue>2025</maxValue>
+ <name>GENERAL_DATE</name>
+ </column>
+ <column>
+ <type>DECIMAL</type>
+ <dataSequence>RANDOM</dataSequence>
+ <minValue>0</minValue>
+ <maxValue>1</maxValue>
+
+ <!-- Precision is limited to 18 -->
+ <precision>18</precision>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>90</nullChance>
+ <name>GENERAL_DECIMAL</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <dataSequence>RANDOM</dataSequence>
+ <minValue>1</minValue>
+ <maxValue>50000000</maxValue>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>100</nullChance>
+ <name>GENERAL_INTEGER</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <userDefined>true</userDefined>
+ <dataSequence>LIST</dataSequence>
+ <length>15</length>
+ <name>TENANT_ID</name>
+ <valuelist>
+ <datavalue distribution="40">
+ <value>00Dxx0000001gER</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>00Dxx0000001gES</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>00Dxx0000001gET</value>
+ </datavalue>
+ <datavalue distribution="15">
+ <value>00Dxx0000001gEU</value>
+ </datavalue>
+ <datavalue distribution="5">
+ <value>00Dxx0000001gEV</value>
+ </datavalue>
+ </valuelist>
+ </column>
+ <column>
+ <type>DATE</type>
+ <userDefined>true</userDefined>
+ <dataSequence>LIST</dataSequence>
+ <name>CREATED_DATE</name>
+ <nullChance>0</nullChance>
+ <valuelist>
+ <datavalue distribution="2">
+ <minValue>2014-08-31 00:00:00.000</minValue>
+ <maxValue>2014-09-01 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-01 00:00:00.000</minValue>
+ <maxValue>2014-09-02 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-02 00:00:00.000</minValue>
+ <maxValue>2014-09-03 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-03 00:00:00.000</minValue>
+ <maxValue>2014-09-04 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-04 00:00:00.000</minValue>
+ <maxValue>2014-09-05 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-05 00:00:00.000</minValue>
+ <maxValue>2014-09-06 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-06 00:00:00.000</minValue>
+ <maxValue>2014-09-07 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-07 00:00:00.000</minValue>
+ <maxValue>2014-09-08 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-08 00:00:00.000</minValue>
+ <maxValue>2014-09-09 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-09 00:00:00.000</minValue>
+ <maxValue>2014-09-10 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-10 00:00:00.000</minValue>
+ <maxValue>2014-09-11 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-11 00:00:00.000</minValue>
+ <maxValue>2014-09-12 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-12 00:00:00.000</minValue>
+ <maxValue>2014-09-13 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-13 00:00:00.000</minValue>
+ <maxValue>2014-09-14 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-14 00:00:00.000</minValue>
+ <maxValue>2014-09-15 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-15 00:00:00.000</minValue>
+ <maxValue>2014-09-16 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-16 00:00:00.000</minValue>
+ <maxValue>2014-09-17 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-17 00:00:00.000</minValue>
+ <maxValue>2014-09-18 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-18 00:00:00.000</minValue>
+ <maxValue>2014-09-19 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-19 00:00:00.000</minValue>
+ <maxValue>2014-09-20 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-20 00:00:00.000</minValue>
+ <maxValue>2014-09-21 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-21 00:00:00.000</minValue>
+ <maxValue>2014-09-22 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-22 00:00:00.000</minValue>
+ <maxValue>2014-09-23 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-23 00:00:00.000</minValue>
+ <maxValue>2014-09-24 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-24 00:00:00.000</minValue>
+ <maxValue>2014-09-25 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-25 00:00:00.000</minValue>
+ <maxValue>2014-09-26 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-26 00:00:00.000</minValue>
+ <maxValue>2014-09-27 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-27 00:00:00.000</minValue>
+ <maxValue>2014-09-28 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-28 00:00:00.000</minValue>
+ <maxValue>2014-09-29 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-29 00:00:00.000</minValue>
+ <maxValue>2014-09-30 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-09-30 00:00:00.000</minValue>
+ <maxValue>2014-10-01 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-01 00:00:00.000</minValue>
+ <maxValue>2014-10-02 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-02 00:00:00.000</minValue>
+ <maxValue>2014-10-03 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-03 00:00:00.000</minValue>
+ <maxValue>2014-10-04 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-04 00:00:00.000</minValue>
+ <maxValue>2014-10-05 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-05 00:00:00.000</minValue>
+ <maxValue>2014-10-06 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-06 00:00:00.000</minValue>
+ <maxValue>2014-10-07 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-07 00:00:00.000</minValue>
+ <maxValue>2014-10-08 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-08 00:00:00.000</minValue>
+ <maxValue>2014-10-09 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-09 00:00:00.000</minValue>
+ <maxValue>2014-10-10 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-10 00:00:00.000</minValue>
+ <maxValue>2014-10-11 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-11 00:00:00.000</minValue>
+ <maxValue>2014-10-12 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-12 00:00:00.000</minValue>
+ <maxValue>2014-10-13 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-13 00:00:00.000</minValue>
+ <maxValue>2014-10-14 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-14 00:00:00.000</minValue>
+ <maxValue>2014-10-15 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-15 00:00:00.000</minValue>
+ <maxValue>2014-10-16 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-16 00:00:00.000</minValue>
+ <maxValue>2014-10-17 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-17 00:00:00.000</minValue>
+ <maxValue>2014-10-18 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-18 00:00:00.000</minValue>
+ <maxValue>2014-10-19 00:00:00.000</maxValue>
+ </datavalue>
+ <datavalue distribution="2">
+ <minValue>2014-10-19 00:00:00.000</minValue>
+ <maxValue>2014-10-20 00:00:00.000</maxValue>
+ </datavalue>
+ </valuelist>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.PHERF_PROD_TEST_UNSALTED" rowCount="100" name="readWriteScenario">
+ <!-- Scenario level rule overrides will be unsupported in V1.
+ You can use the general datamappings in the mean time-->
+ <dataOverride>
+ <column>
+ <type>VARCHAR</type>
+ <userDefined>true</userDefined>
+ <length>15</length>
+ <dataSequence>LIST</dataSequence>
+ <valueList>
+ <datavalue>
+ <value>00Dxx0000001gER</value>
+ </datavalue>
+ <datavalue>
+ <value>00Dxx0000001gES</value>
+ </datavalue>
+ <datavalue>
+ <value>00Dxx0000001gET</value>
+ </datavalue>
+ </valueList>
+ <name>TENANT_ID</name>
+ </column>
+ </dataOverride>
+
+ <preScenarioDdls>
+ <ddl statement="CREATE INDEX IF NOT EXISTS IDX_DIVISION ON PHERF.PHERF_PROD_TEST_UNSALTED (DIVISION)"/>
+ </preScenarioDdls>
+
+ <postScenarioDdls>
+ <ddl statement="CREATE INDEX IF NOT EXISTS IDX_OLDVAL_STRING ON PHERF.PHERF_PROD_TEST_UNSALTED (OLDVAL_STRING)"/>
+ <ddl statement="CREATE INDEX IF NOT EXISTS IDX_CONNECTION_ID ON PHERF.PHERF_PROD_TEST_UNSALTED (CONNECTION_ID)"/>
+ </postScenarioDdls>
+
+ <writeParams executionDurationInMs="10000">
+ <!--
+ Number of writer it insert into the threadpool
+ -->
+ <writerThreadCount>5</writerThreadCount>
+
+ <!--
+ Time in Ms that each thread will sleep between batch writes. This helps to
+ throttle writers.
+ -->
+ <threadSleepDuration>10</threadSleepDuration>
+
+ <batchSize>100</batchSize>
+ </writeParams>
+ <!--Minimum of executionDurationInMs or numberOfExecutions. Which ever is reached first -->
+ <querySet concurrency="1" executionType="PARALLEL" executionDurationInMs="60000"
+ numberOfExecutions="100">
+ <!-- Aggregate queries on a per tenant basis -->
+ <query tenantId="00Dxx0000001gER"
+ ddl="CREATE VIEW IF NOT EXISTS PHERF.PHERF_TEST_VIEW_UNSALTED AS SELECT * FROM PHERF.PHERF_PROD_TEST_UNSALTED"
+ statement="select count(*) from PHERF.PHERF_TEST_VIEW_UNSALTED"/>
+ </querySet>
+
+ </scenario>
+ <scenario tableName="PHERF.PHERF_PROD_TEST_UNSALTED" rowCount="10" name="readWriteScenario">
+ <dataOverride>
+ <column>
+ <type>VARCHAR</type>
+ <userDefined>true</userDefined>
+ <length>15</length>
+ <dataSequence>LIST</dataSequence>
+ <valueList>
+ <datavalue>
+ <value>00Dxx0000001gER</value>
+ </datavalue>
+ <datavalue>
+ <value>00Dxx0000001gES</value>
+ </datavalue>
+ <datavalue>
+ <value>00Dxx0000001gET</value>
+ </datavalue>
+ </valueList>
+ <name>TENANT_ID</name>
+ </column>
+ </dataOverride>
+
+ <!-- Pre and post scenario indexes -->
+ <preScenarioDdls>
+ <ddl statement="CREATE INDEX IF NOT EXISTS IDX_DIVISION ON PHERF.PHERF_PROD_TEST_UNSALTED (DIVISION)"/>
+ </preScenarioDdls>
+
+ <postScenarioDdls>
+ <ddl statement="CREATE INDEX IF NOT EXISTS IDX_OLDVAL_STRING ON PHERF.PHERF_PROD_TEST_UNSALTED (OLDVAL_STRING)"/>
+ <ddl statement="CREATE INDEX IF NOT EXISTS IDX_CONNECTION_ID ON PHERF.PHERF_PROD_TEST_UNSALTED (CONNECTION_ID)"/>
+ </postScenarioDdls>
+
+ <!--Minimum of executionDurationInMs or numberOfExecutions. Which ever is reached first -->
+ <querySet concurrency="1" executionType="PARALLEL" executionDurationInMs="60000" numberOfExecutions="100">
+ <query statement="select count(*) from PHERF.PHERF_PROD_TEST_UNSALTED WHERE TENANT_ID=[TENANT_ID] AND TENANT_ID=[TENANT_ID]"/>
+ <!-- Aggregate queries on a per tenant basis -->
+ <query tenantId="00Dxx0000001gER"
+ ddl="CREATE VIEW IF NOT EXISTS PHERF.PHERF_TEST_VIEW_UNSALTED AS SELECT * FROM PHERF.PHERF_PROD_TEST_UNSALTED"
+ statement="select count(*) from PHERF.PHERF_TEST_VIEW_UNSALTED"/>
+ <query tenantId="00Dxx0000001gES"
+ ddl="CREATE VIEW IF NOT EXISTS PHERF.PHERF_TEST_VIEW_UNSALTED AS SELECT * FROM PHERF.PHERF_PROD_TEST_UNSALTED"
+ statement="select /*+ SMALL*/ count(*) from PHERF.PHERF_TEST_VIEW_UNSALTED"/>
+ </querySet>
+
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
index 876f10b..d04801b 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/Pherf.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Properties;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.CommandLineParser;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.DefaultParser;
@@ -33,6 +34,8 @@ import org.apache.phoenix.thirdparty.org.apache.commons.cli.Options;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.ParseException;
import org.apache.phoenix.pherf.PherfConstants.CompareType;
import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.configuration.XMLConfigParser;
import org.apache.phoenix.pherf.jmx.MonitorManager;
import org.apache.phoenix.pherf.result.ResultUtil;
@@ -40,6 +43,7 @@ import org.apache.phoenix.pherf.schema.SchemaReader;
import org.apache.phoenix.pherf.util.GoogleChartGenerator;
import org.apache.phoenix.pherf.util.PhoenixUtil;
import org.apache.phoenix.pherf.util.ResourceList;
+import org.apache.phoenix.pherf.workload.mt.MultiTenantWorkload;
import org.apache.phoenix.pherf.workload.QueryExecutor;
import org.apache.phoenix.pherf.workload.Workload;
import org.apache.phoenix.pherf.workload.WorkloadExecutor;
@@ -60,10 +64,14 @@ public class Pherf {
"HBase Zookeeper address for connection. Default: localhost");
options.addOption("q", "query", false, "Executes multi-threaded query sets");
options.addOption("listFiles", false, "List available resource files");
+ options.addOption("mt", "multi-tenant", false,
+ "Multi tenanted workloads based on load profiles.");
options.addOption("l", "load", false,
"Pre-loads data according to specified configuration values.");
options.addOption("scenarioFile", true,
"Regex or file name for the Test Scenario configuration .xml file to use.");
+ options.addOption("scenarioName", true,
+ "Regex or scenario name from the Test Scenario configuration .xml file to use.");
options.addOption("drop", true, "Regex drop all tables with schema name as PHERF. "
+ "\nExample drop Event tables: -drop .*(EVENT).* Drop all: -drop .* or -drop all");
options.addOption("schemaFile", true,
@@ -99,10 +107,12 @@ public class Pherf {
private final String zookeeper;
private final String scenarioFile;
+ private final String scenarioName;
private final String schemaFile;
private final String queryHint;
private final Properties properties;
private final boolean preLoadData;
+ private final boolean multiTenantWorkload;
private final String dropPherfTablesRegEx;
private final boolean executeQuerySets;
private final boolean isFunctional;
@@ -148,6 +158,7 @@ public class Pherf {
properties.setProperty(PherfConstants.LOG_PER_NROWS_NAME, getLogPerNRow(command));
preLoadData = command.hasOption("l");
+ multiTenantWorkload = command.hasOption("mt");
executeQuerySets = command.hasOption("q");
zookeeper = command.getOptionValue("z", "localhost");
queryHint = command.getOptionValue("hint", null);
@@ -157,6 +168,8 @@ public class Pherf {
writeRuntimeResults = !command.hasOption("disableRuntimeResult");
scenarioFile =
command.hasOption("scenarioFile") ? command.getOptionValue("scenarioFile") : null;
+ scenarioName =
+ command.hasOption("scenarioName") ? command.getOptionValue("scenarioName") : null;
schemaFile = command.hasOption("schemaFile") ? command.getOptionValue("schemaFile") : null;
rowCountOverride = Integer.parseInt(command.getOptionValue("rowCountOverride", "0"));
generateStatistics = command.hasOption("stats") ? GeneratePhoenixStats.YES : GeneratePhoenixStats.NO;
@@ -259,8 +272,6 @@ public class Pherf {
new GoogleChartGenerator(compareResults, compareType).readAndRender();
return;
}
-
- XMLConfigParser parser = new XMLConfigParser(scenarioFile);
// Drop tables with PHERF schema and regex comparison
if (null != dropPherfTablesRegEx) {
@@ -270,12 +281,6 @@ public class Pherf {
phoenixUtil.deleteTables(dropPherfTablesRegEx);
}
- if (monitor) {
- monitorManager =
- new MonitorManager(Integer.parseInt(
- properties.getProperty("pherf.default.monitorFrequency")));
- workloadExecutor.add(monitorManager);
- }
if (applySchema) {
LOGGER.info("\nStarting to apply schema...");
@@ -287,18 +292,54 @@ public class Pherf {
reader.applySchema();
}
+ // If no scenario file specified then we are done.
+ if (scenarioFile == null) {
+ return;
+ }
+
+ XMLConfigParser parser = new XMLConfigParser(scenarioFile);
+ if (monitor) {
+ monitorManager =
+ new MonitorManager(Integer.parseInt(
+ properties.getProperty("pherf.default.monitorFrequency")));
+ workloadExecutor.add(monitorManager);
+ }
+
// Schema and Data Load
- if (preLoadData) {
+ if (preLoadData || multiTenantWorkload) {
LOGGER.info("\nStarting Data Load...");
- Workload workload = new WriteWorkload(parser, generateStatistics);
+ List<Workload> newWorkloads = Lists.newArrayList();
try {
- workloadExecutor.add(workload);
+ if (multiTenantWorkload) {
+ for (DataModel model : parser.getDataModels()) {
+ for (Scenario scenario : model.getScenarios()) {
+ if ((scenarioName != null) && (scenarioName.compareTo(scenario.getName()) != 0)) {
+ continue;
+ }
+ Workload workload = new MultiTenantWorkload(phoenixUtil,
+ model, scenario, properties);
+ newWorkloads.add(workload);
+ }
+ }
+ } else {
+ newWorkloads.add(new WriteWorkload(parser, generateStatistics));
+ }
+
+ if (newWorkloads.isEmpty()) {
+ throw new IllegalArgumentException("Found no new workload");
+ }
+
+ for (Workload workload : newWorkloads) {
+ workloadExecutor.add(workload);
+ }
// Wait for dataLoad to complete
- workloadExecutor.get(workload);
+ workloadExecutor.get();
} finally {
- if (null != workload) {
- workload.complete();
+ if (!newWorkloads.isEmpty()) {
+ for (Workload workload : newWorkloads) {
+ workload.complete();
+ }
}
}
} else {
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
index 2a7a9b9..c749d15 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java
@@ -78,6 +78,12 @@ public class PherfConstants {
public static final int MONITOR_FREQUENCY = 5000;
public static final String MONITOR_FILE_NAME = "STATS_MONITOR";
+ public static final String NUM_SEQUENTIAL_ITERATIONS_PROP_KEY = "pherf.mt.sequential.iterations";
+ public static final String NUM_SEQUENTIAL_EXECUTION_TYPE_PROP_KEY = "pherf.mt.sequential.type";
+ public static final String HANDLERS_PER_SCENARIO_PROP_KEY = "pherf.mt.handlers_per_scenario";
+ public static final String MT_HANDLER_START_RENDEZVOUS_PROP_KEY = "pherf.mt.handlers_start_rendezvous";
+ public static final String MT_HANDLER_RESULTS_RENDEZVOUS_PROP_KEY = "pherf.mt.handlers_results_rendezvous";
+
private PherfConstants() {
}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java
index 129bdc2..3bbe728 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/DataTypeMapping.java
@@ -30,7 +30,9 @@ public enum DataTypeMapping {
VARCHAR_ARRAY("VARCHAR ARRAY", Types.ARRAY),
VARBINARY("VARBINARY", Types.VARBINARY),
TIMESTAMP("TIMESTAMP", Types.TIMESTAMP),
+ BOOLEAN("BOOLEAN", Types.BOOLEAN),
BIGINT("BIGINT", Types.BIGINT),
+ UNSIGNED_INT("UNSIGNED_INT", Types.INTEGER),
TINYINT("TINYINT", Types.TINYINT);
private final String sType;
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
index 3af8c3f..b60508d 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Ddl.java
@@ -23,7 +23,8 @@ import javax.xml.bind.annotation.XmlAttribute;
public class Ddl {
private String statement;
private String tableName;
-
+ private boolean useGlobalConnection;
+
public Ddl() {
}
@@ -55,7 +56,16 @@ public class Ddl {
public void setTableName(String tableName) {
this.tableName = tableName;
}
-
+
+ @XmlAttribute
+ public boolean isUseGlobalConnection() {
+ return useGlobalConnection;
+ }
+
+ public void setUseGlobalConnection(boolean useGlobalConnection) {
+ this.useGlobalConnection = useGlobalConnection;
+ }
+
public String toString(){
if (statement.contains("?")) {
return statement.replace("?", tableName);
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/IdleTime.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/IdleTime.java
new file mode 100644
index 0000000..37d6e15
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/IdleTime.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.configuration;
+
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlType;
+
+@XmlType
+public class IdleTime {
+
+ private String id;
+ private long idleTime = 0;
+
+ @XmlAttribute
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @XmlAttribute
+ public long getIdleTime() {
+ return idleTime;
+ }
+
+ public void setIdleTime(long idleTime) {
+ this.idleTime = idleTime;
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java
new file mode 100644
index 0000000..c66bbee
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/LoadProfile.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.configuration;
+
+import javax.xml.bind.annotation.XmlType;
+import java.util.List;
+
+@XmlType
+public class LoadProfile {
+ private static final int MIN_BATCH_SIZE = 1;
+ private static final String DEFAULT_TENANT_ID_FMT = "T%s%08d";
+ private static final int DEFAULT_GROUP_ID_LEN = 6;
+ private static final int DEFAULT_TENANT_ID_LEN = 15;
+
+ // Holds the batch size to be used in upserts.
+ private int batchSize;
+ // Holds the number of operations to be generated.
+ private long numOperations;
+ /**
+ * Holds the format to be used when generating tenantIds.
+ * TenantId format should typically have 2 parts -
+ * 1. string fmt - that hold the tenant group id.
+ * 2. int fmt - that holds a random number between 1 and max tenants
+ * for e.g DEFAULT_TENANT_ID_FMT = "T%s%08d";
+ *
+ * When the Tenant Group is configured to use a global connection,
+ * for now this is modelled as a special tenant whose id will translate to "TGLOBAL00000001"
+ * since the group id => "GLOBAL" and num tenants = 1.
+ * For now this is a hack/temporary workaround.
+ *
+ * TODO :
+ * Ideally it needs to be built into the framework and injected during event generation.
+ */
+ private String tenantIdFormat;
+ private int groupIdLength;
+ private int tenantIdLength;
+ // Holds the desired tenant distribution for this load.
+ private List<TenantGroup> tenantDistribution;
+ // Holds the desired operation distribution for this load.
+ private List<OperationGroup> opDistribution;
+
+ public LoadProfile() {
+ this.batchSize = MIN_BATCH_SIZE;
+ this.numOperations = Long.MAX_VALUE;
+ this.tenantIdFormat = DEFAULT_TENANT_ID_FMT;
+ this.tenantIdLength = DEFAULT_TENANT_ID_LEN;
+ this.groupIdLength = DEFAULT_GROUP_ID_LEN;
+ }
+
+ public String getTenantIdFormat() {
+ return tenantIdFormat;
+ }
+
+ public void setTenantIdFormat(String tenantIdFormat) {
+ this.tenantIdFormat = tenantIdFormat;
+ }
+
+ public int getTenantIdLength() {
+ return tenantIdLength;
+ }
+
+ public void setTenantIdLength(int tenantIdLength) {
+ this.tenantIdLength = tenantIdLength;
+ }
+
+ public int getGroupIdLength() {
+ return groupIdLength;
+ }
+
+ public void setGroupIdLength(int groupIdLength) {
+ this.groupIdLength = groupIdLength;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public long getNumOperations() {
+ return numOperations;
+ }
+
+ public void setNumOperations(long numOperations) {
+ this.numOperations = numOperations;
+ }
+
+ public List<TenantGroup> getTenantDistribution() {
+ return tenantDistribution;
+ }
+
+ public void setTenantDistribution(List<TenantGroup> tenantDistribution) {
+ this.tenantDistribution = tenantDistribution;
+ }
+
+ public List<OperationGroup> getOpDistribution() {
+ return opDistribution;
+ }
+
+ public void setOpDistribution(List<OperationGroup> opDistribution) {
+ this.opDistribution = opDistribution;
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/OperationGroup.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/OperationGroup.java
new file mode 100644
index 0000000..31545b2
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/OperationGroup.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.configuration;
+
+import javax.xml.bind.annotation.XmlAttribute;
+
+public class OperationGroup {
+ private String id;
+ private int weight;
+
+ @XmlAttribute
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @XmlAttribute
+ public int getWeight() {
+ return weight;
+ }
+
+ public void setWeight(int weight) {
+ this.weight = weight;
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java
index 5f28134..c47798c 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Query.java
@@ -18,23 +18,23 @@
package org.apache.phoenix.pherf.configuration;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.apache.phoenix.pherf.rules.RulesApplier;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlType;
-
-import org.apache.phoenix.pherf.rules.RulesApplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
@XmlType
public class Query {
+ private String id;
+ private String queryGroup;
+ private String tenantId;
private String statement;
private Long expectedAggregateRowCount;
- private String tenantId;
private String ddl;
- private String queryGroup;
- private String id;
+ private boolean useGlobalConnection;
private Pattern pattern;
private long timeoutDuration = Long.MAX_VALUE;
@@ -51,20 +51,24 @@ public class Query {
public String getStatement() {
return statement;
}
-
- public String getDynamicStatement(RulesApplier ruleApplier, Scenario scenario) throws Exception {
- String ret = this.statement;
- String needQuotes = "";
- Matcher m = pattern.matcher(ret);
- while(m.find()) {
- String dynamicField = m.group(0).replace("[", "").replace("]", "");
- Column dynamicColumn = ruleApplier.getRule(dynamicField, scenario);
- needQuotes = (dynamicColumn.getType() == DataTypeMapping.CHAR || dynamicColumn
- .getType() == DataTypeMapping.VARCHAR) ? "'" : "";
- ret = ret.replace("[" + dynamicField + "]",
- needQuotes + ruleApplier.getDataValue(dynamicColumn).getValue() + needQuotes);
- }
- return ret;
+
+ public String getDynamicStatement(RulesApplier ruleApplier, Scenario scenario)
+ throws Exception {
+ String ret = this.statement;
+ String needQuotes = "";
+ Matcher m = pattern.matcher(ret);
+ while (m.find()) {
+ String dynamicField = m.group(0).replace("[", "").replace("]", "");
+ Column dynamicColumn = ruleApplier.getRule(dynamicField, scenario);
+ needQuotes =
+ (dynamicColumn.getType() == DataTypeMapping.CHAR
+ || dynamicColumn.getType() == DataTypeMapping.VARCHAR) ? "'" : "";
+ ret =
+ ret.replace("[" + dynamicField + "]",
+ needQuotes + ruleApplier.getDataValue(dynamicColumn).getValue()
+ + needQuotes);
+ }
+ return ret;
}
public void setStatement(String statement) {
@@ -160,6 +164,14 @@ public class Query {
this.id = id;
}
+ @XmlAttribute
+ public boolean isUseGlobalConnection() {
+ return useGlobalConnection;
+ }
+
+ public void setUseGlobalConnection(boolean useGlobalConnection) {
+ this.useGlobalConnection = useGlobalConnection;
+ }
@XmlAttribute
public long getTimeoutDuration() {
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
index 53b2d25..57175a7 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
@@ -36,15 +36,20 @@ public class Scenario {
private String tableName;
private int rowCount;
private Map<String, String> phoenixProperties;
+ private WriteParams writeParams = null;
private DataOverride dataOverride;
private List<QuerySet> querySet = new ArrayList<>();
- private WriteParams writeParams = null;
+ private List<Upsert> upsertSet = new ArrayList<>();
+ private List<IdleTime> idleTimes = new ArrayList<>();
+ private List<UserDefined> udfs = new ArrayList<>();
+ private LoadProfile loadProfile = null;
+
private String name;
+ private String generatorName;
private String tenantId;
private List<Ddl> preScenarioDdls;
private List<Ddl> postScenarioDdls;
-
-
+
public Scenario() {
}
@@ -85,6 +90,20 @@ public class Scenario {
}
/**
+ * Generator name for a scenario
+ *
+ * @return
+ */
+ @XmlAttribute()
+ public String getGeneratorName() {
+ return generatorName;
+ }
+
+ public void setGeneratorName(String name) {
+ this.generatorName = name;
+ }
+
+ /**
* Row count for a table
*
* @return
@@ -194,6 +213,7 @@ public class Scenario {
this.writeParams = writeParams;
}
+
@Override
public String toString() {
StringBuilder stringBuilder = new StringBuilder();
@@ -232,4 +252,43 @@ public class Scenario {
public void setPostScenarioDdls(List<Ddl> postScenarioDdls) {
this.postScenarioDdls = postScenarioDdls;
}
+
+ public List<Upsert> getUpserts() {
+ return upsertSet;
+ }
+
+ @XmlElementWrapper(name = "upserts")
+ @XmlElement(name = "upsert")
+ public void setUpserts(List<Upsert> upsertSet) {
+ this.upsertSet = upsertSet;
+ }
+
+ public List<IdleTime> getIdleTimes() {
+ return idleTimes;
+ }
+
+ @XmlElementWrapper(name = "idleTimes")
+ @XmlElement(name = "idleTime")
+ public void setIdleTimes(List<IdleTime> idleTimes) {
+ this.idleTimes = idleTimes;
+ }
+
+ public List<UserDefined> getUdfs() {
+ return udfs;
+ }
+
+ @XmlElementWrapper(name = "udfs")
+ @XmlElement(name = "udf")
+ public void setUdfs(List<UserDefined> udfs) {
+ this.udfs = udfs;
+ }
+
+
+ public LoadProfile getLoadProfile() {
+ return loadProfile;
+ }
+
+ public void setLoadProfile(LoadProfile loadProfile) {
+ this.loadProfile = loadProfile;
+ }
}
\ No newline at end of file
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java
new file mode 100644
index 0000000..d066cd5
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/TenantGroup.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.configuration;
+
+import javax.xml.bind.annotation.XmlAttribute;
+
+public class TenantGroup {
+ public static final String DEFAULT_GLOBAL_ID = "GLOBAL";
+ private String id;
+ private int weight;
+ private int numTenants;
+ private boolean useGlobalConnection;
+
+ @XmlAttribute
+ public String getId() {
+ return useGlobalConnection ? DEFAULT_GLOBAL_ID: id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @XmlAttribute
+ public int getWeight() {
+ return useGlobalConnection ? 100 : weight;
+ }
+
+ public void setWeight(int weight) {
+ this.weight = weight;
+ }
+
+ @XmlAttribute
+ public int getNumTenants() { return useGlobalConnection ? 1 : numTenants; }
+
+ public void setNumTenants(int numTenants) { this.numTenants = numTenants; }
+
+ @XmlAttribute
+ public boolean isUseGlobalConnection() {
+ return useGlobalConnection;
+ }
+
+ public void setUseGlobalConnection(boolean useGlobalConnection) {
+ this.useGlobalConnection = useGlobalConnection;
+ }
+
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Upsert.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Upsert.java
new file mode 100644
index 0000000..8d3cef6
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Upsert.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.configuration;
+
+import org.apache.phoenix.pherf.rules.RulesApplier;
+import com.google.common.collect.Lists;
+
+import javax.xml.bind.annotation.XmlAttribute;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Upsert {
+
+ private String id;
+ private String upsertGroup;
+ private String statement;
+ private List<Column> column;
+ private boolean useGlobalConnection;
+ private Pattern pattern;
+ private long timeoutDuration = Long.MAX_VALUE;
+
+ public Upsert() {
+ pattern = Pattern.compile("\\[.*?\\]");
+ }
+
+ public String getDynamicStatement(RulesApplier ruleApplier, Scenario scenario)
+ throws Exception {
+ String ret = this.statement;
+ String needQuotes = "";
+ Matcher m = pattern.matcher(ret);
+ while (m.find()) {
+ String dynamicField = m.group(0).replace("[", "").replace("]", "");
+ Column dynamicColumn = ruleApplier.getRule(dynamicField, scenario);
+ needQuotes =
+ (dynamicColumn.getType() == DataTypeMapping.CHAR
+ || dynamicColumn.getType() == DataTypeMapping.VARCHAR) ? "'" : "";
+ ret = ret.replace("[" + dynamicField + "]",
+ needQuotes + ruleApplier.getDataValue(dynamicColumn).getValue()
+ + needQuotes);
+ }
+ return ret;
+ }
+
+ /**
+ * upsertGroup attribute is just a string value to help correlate upserts across sets or files.
+ * This helps to make sense of reporting results.
+ *
+ * @return the group id
+ */
+ @XmlAttribute
+ public String getUpsertGroup() {
+ return upsertGroup;
+ }
+
+ public void setUpsertGroup(String upsertGroup) {
+ this.upsertGroup = upsertGroup;
+ }
+
+
+ /**
+ * Upsert ID, Use UUID if none specified
+ *
+ * @return
+ */
+ @XmlAttribute
+ public String getId() {
+ if (null == this.id) {
+ this.id = java.util.UUID.randomUUID().toString();
+ }
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public List<Column> getColumn() {
+ if (column == null) return Lists.newArrayList();
+ return column;
+ }
+
+ public void setColumn(List<Column> column) {
+ this.column = column;
+ }
+
+ @XmlAttribute
+ public boolean isUseGlobalConnection() {
+ return useGlobalConnection;
+ }
+
+ public void setUseGlobalConnection(boolean useGlobalConnection) {
+ this.useGlobalConnection = useGlobalConnection;
+ }
+
+ @XmlAttribute
+ public long getTimeoutDuration() {
+ return this.timeoutDuration;
+ }
+
+ public void setTimeoutDuration(long timeoutDuration) {
+ this.timeoutDuration = timeoutDuration;
+ }
+
+ public String getStatement() {
+ return statement;
+ }
+
+ public void setStatement(String statement) {
+ // normalize statement - merge all consecutive spaces into one
+ this.statement = statement.replaceAll("\\s+", " ");
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/UserDefined.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/UserDefined.java
new file mode 100644
index 0000000..8350d57
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/UserDefined.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.configuration;
+
+import javax.xml.bind.annotation.XmlAttribute;
+import javax.xml.bind.annotation.XmlType;
+import java.util.List;
+
+@XmlType
+public class UserDefined {
+ String id;
+ String clazzName;
+ List<String> args;
+
+ @XmlAttribute
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getClazzName() {
+ return clazzName;
+ }
+
+ public void setClazzName(String clazzName) {
+ this.clazzName = clazzName;
+ }
+
+ public List<String> getArgs() {
+ return args;
+ }
+
+ public void setArgs(List<String> args) {
+ this.args = args;
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/jmx/MonitorManager.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/jmx/MonitorManager.java
index 5c434d8..5800a20 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/jmx/MonitorManager.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/jmx/MonitorManager.java
@@ -172,8 +172,8 @@ public class MonitorManager implements Workload {
/**
* This method should really only be used for testing
*
- * @return List < {@link org.apache.phoenix.pherf.result.Result} >
- * @throws IOException
+ * @return {@code List < org.apache.phoenix.pherf.result.Result > }
+ * @throws Exception
*/
public synchronized List<Result> readResults() throws Exception {
ResultHandler handler = null;
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/Result.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/Result.java
index 158ed11..93225d7 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/Result.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/Result.java
@@ -36,7 +36,7 @@ public class Result {
* @param type {@link org.apache.phoenix.pherf.result.file.ResultFileDetails} Currently unused, but gives metadata about the
* contents of the result.
* @param header Used for CSV, otherwise pass null. For CSV pass comma separated string of header fields.
- * @param messageValues List<{@link ResultValue} All fields combined represent the data
+ * @param messageValues {@code List<ResultValue> } All fields combined represent the data
* for a row to be written.
*/
public Result(ResultFileDetails type, String header, List<ResultValue> messageValues) {
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java
index 1cf740e..91db782 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/result/ResultManager.java
@@ -121,7 +121,7 @@ public class ResultManager {
/**
* Write a combined set of results for each result in the list.
*
- * @param dataModelResults List<{@link DataModelResult > </>}
+ * @param dataModelResults {@code List<DataModelResult > }
* @throws Exception
*/
public synchronized void write(List<DataModelResult> dataModelResults, RulesApplier rulesApplier) throws Exception {
@@ -145,7 +145,6 @@ public class ResultManager {
/**
* Allows for flushing all the {@link org.apache.phoenix.pherf.result.ResultHandler}
- * @throws Exception
*/
public synchronized void flush(){
for (ResultHandler handler : resultHandlers) {
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RuleBasedDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RuleBasedDataGenerator.java
index 24ecd20..d68e468 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RuleBasedDataGenerator.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RuleBasedDataGenerator.java
@@ -24,7 +24,7 @@ public interface RuleBasedDataGenerator {
* Get data value based on the rules
* Implementations should be thread safe as multiple theads will call it in parallel
*
- * @return {@link org.apache.phoenix.pherf.rules.DataValue} Container Type --> Value mapping
+ * @return {@link org.apache.phoenix.pherf.rules.DataValue} {@code Container Type --> Value } mapping
*/
DataValue getDataValue();
}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
index b58e2f7..e882325 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.pherf.rules;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.math3.random.RandomDataGenerator;
import org.apache.phoenix.pherf.PherfConstants;
@@ -34,14 +35,12 @@ import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.text.SimpleDateFormat;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicLong;
public class RulesApplier {
private static final Logger LOGGER = LoggerFactory.getLogger(RulesApplier.class);
- private static final AtomicLong COUNTER = new AtomicLong(0);
// Used to bail out of random distribution if it takes too long
// This should never happen when distributions add up to 100
@@ -51,14 +50,39 @@ public class RulesApplier {
private final Random rndVal;
private final RandomDataGenerator randomDataGenerator;
+ private final DataModel dataModel;
private final XMLConfigParser parser;
private final List<Map> modelList;
private final Map<String, Column> columnMap;
private String cachedScenarioOverrideName;
private Map<DataTypeMapping, List> scenarioOverrideMap;
- private Map<Column,RuleBasedDataGenerator> columnRuleBasedDataGeneratorMap = new HashMap<>();
+ private ConcurrentHashMap<String,RuleBasedDataGenerator> columnRuleBasedDataGeneratorMap = new ConcurrentHashMap<>();
+ // Since rules are only relevant for a given data model,
+ // added a constructor to support a single data model => RulesApplier(DataModel model)
+
+ // We should deprecate the RulesApplier(XMLConfigParser parser) constructor,
+ // since a parser can have multiple data models (all the models found on the classpath)
+ // it implies that the rules apply to all the data models the parser holds
+ // which can be confusing to the user of this class.
+ //
+
+ public RulesApplier(DataModel model) {
+ this(model, EnvironmentEdgeManager.currentTimeMillis());
+ }
+
+ public RulesApplier(DataModel model, long seed) {
+ this.parser = null;
+ this.dataModel = model;
+ this.modelList = new ArrayList<Map>();
+ this.columnMap = new HashMap<String, Column>();
+ this.rndNull = new Random(seed);
+ this.rndVal = new Random(seed);
+ this.randomDataGenerator = new RandomDataGenerator();
+ this.cachedScenarioOverrideName = null;
+ populateModelList();
+ }
public RulesApplier(XMLConfigParser parser) {
this(parser, EnvironmentEdgeManager.currentTimeMillis());
@@ -66,6 +90,7 @@ public class RulesApplier {
public RulesApplier(XMLConfigParser parser, long seed) {
this.parser = parser;
+ this.dataModel = null;
this.modelList = new ArrayList<Map>();
this.columnMap = new HashMap<String, Column>();
this.rndNull = new Random(seed);
@@ -116,10 +141,10 @@ public class RulesApplier {
public DataValue getDataForRule(Scenario scenario, Column phxMetaColumn) throws Exception {
// TODO Make a Set of Rules that have already been applied so that so we don't generate for every value
- List<Scenario> scenarios = parser.getScenarios();
+ List<Scenario> scenarios = dataModel != null ? dataModel.getScenarios() : parser.getScenarios();
DataValue value = null;
if (scenarios.contains(scenario)) {
- LOGGER.debug("We found a correct Scenario");
+ LOGGER.debug("We found a correct Scenario" + scenario.getName());
Map<DataTypeMapping, List> overrideRuleMap = this.getCachedScenarioOverrides(scenario);
@@ -140,16 +165,16 @@ public class RulesApplier {
List<Column> ruleList = ruleMap.get(phxMetaColumn.getType());
// Make sure Column from Phoenix Metadata matches a rule column
- if (ruleList.contains(phxMetaColumn)) {
+ if (ruleList != null && ruleList.contains(phxMetaColumn)) {
// Generate some random data based on this rule
LOGGER.debug("We found a correct column rule");
Column columnRule = getColumnForRule(ruleList, phxMetaColumn);
value = getDataValue(columnRule);
} else {
- LOGGER.warn("Attempted to apply rule to data, but could not find a rule to match type:"
- + phxMetaColumn.getType()
- );
+ LOGGER.warn(String.format("Attempted to apply rule to data, "
+ + "but could not find a rule to match type %s on %s",
+ phxMetaColumn.getType(), phxMetaColumn.getName()));
}
}
@@ -161,9 +186,9 @@ public class RulesApplier {
* Get data value based on the supplied rule
*
* @param column {@link org.apache.phoenix.pherf.configuration.Column} Column rule to get data for
- * @return {@link org.apache.phoenix.pherf.rules.DataValue} Container Type --> Value mapping
+ * @return {@link org.apache.phoenix.pherf.rules.DataValue} {@code Container Type --> Value mapping }
*/
- public DataValue getDataValue(Column column) throws Exception{
+ public DataValue getDataValue(Column column) throws Exception {
DataValue data = null;
String prefix = "";
int length = column.getLength();
@@ -190,15 +215,14 @@ public class RulesApplier {
case VARBINARY:
case CHAR:
// Use the specified data values from configs if they exist
- if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
+ if (DataSequence.SEQUENTIAL.equals(column.getDataSequence())) {
+ RuleBasedDataGenerator generator = getRuleBasedDataGeneratorForColumn(column);
+ data = generator.getDataValue();
+ } else if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
data = pickDataValueFromList(dataValues);
} else {
Preconditions.checkArgument(length > 0, "length needs to be > 0");
- if (column.getDataSequence() == DataSequence.SEQUENTIAL) {
- data = getSequentialVarcharDataValue(column);
- } else {
- data = getRandomDataValue(column);
- }
+ data = getRandomDataValue(column);
}
break;
case VARCHAR_ARRAY:
@@ -269,6 +293,9 @@ public class RulesApplier {
data = pickDataValueFromList(dataValues);
// Check if date has right format or not
data.setValue(checkDatePattern(data.getValue()));
+ } else if(DataSequence.SEQUENTIAL.equals(column.getDataSequence())) {
+ RuleBasedDataGenerator generator = getRuleBasedDataGeneratorForColumn(column);
+ data = generator.getDataValue();
} else if (column.getUseCurrentDate() != true){
int minYear = (int) column.getMinValue();
int maxYear = (int) column.getMaxValue();
@@ -422,9 +449,18 @@ public class RulesApplier {
if (!modelList.isEmpty()) {
return;
}
-
- // Support for multiple models, but rules are only relevant each model
- for (DataModel model : parser.getDataModels()) {
+
+ // Since rules are only relevant for a given data model,
+ // added a constructor to support a single data model => RulesApplier(DataModel model)
+
+ // We should deprecate the RulesApplier(XMLConfigParser parser) constructor,
+ // since a parser can have multiple data models (all the models found on the classpath)
+ // it implies that the rules apply to all the data models the parser holds
+ // which can be confusing to the user of this class.
+
+ List<DataModel> models = dataModel != null ?
+ Lists.newArrayList(dataModel) : parser.getDataModels();
+ for (DataModel model : models) {
// Step 1
final Map<DataTypeMapping, List> ruleMap = new HashMap<DataTypeMapping, List>();
@@ -500,28 +536,6 @@ public class RulesApplier {
return ruleAppliedColumn;
}
- /**
- * Add a numerically increasing counter onto the and of a random string.
- * Incremented counter should be thread safe.
- *
- * @param column {@link org.apache.phoenix.pherf.configuration.Column}
- * @return {@link org.apache.phoenix.pherf.rules.DataValue}
- */
- private DataValue getSequentialVarcharDataValue(Column column) {
- DataValue data = null;
- long inc = COUNTER.getAndIncrement();
- String strInc = String.valueOf(inc);
- int paddedLength = column.getLengthExcludingPrefix();
- String strInc1 = StringUtils.leftPad(strInc, paddedLength, "0");
- String strInc2 = StringUtils.right(strInc1, column.getLengthExcludingPrefix());
- String varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc2:
- strInc2;
-
- // Truncate string back down if it exceeds length
- varchar = StringUtils.left(varchar,column.getLength());
- data = new DataValue(column.getType(), varchar);
- return data;
- }
private DataValue getRandomDataValue(Column column) {
String varchar = RandomStringUtils.randomAlphanumeric(column.getLength());
@@ -533,11 +547,39 @@ public class RulesApplier {
}
private RuleBasedDataGenerator getRuleBasedDataGeneratorForColumn(Column column) {
- RuleBasedDataGenerator generator = columnRuleBasedDataGeneratorMap.get(column);
+ RuleBasedDataGenerator generator = columnRuleBasedDataGeneratorMap.get(column.getName());
if(generator == null) {
- //For now we only have one of these, likely this should replace all all the methods
- generator = new SequentialIntegerDataGenerator(column);
- columnRuleBasedDataGeneratorMap.put(column,generator);
+ //For now we only have couple of these, likely this should replace for all the methods
+ switch (column.getType()) {
+ case VARCHAR:
+ case VARBINARY:
+ case CHAR:
+ if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
+ generator = new SequentialListDataGenerator(column);
+ } else {
+ generator = new SequentialVarcharDataGenerator(column);
+ }
+ break;
+ case DATE:
+ case TIMESTAMP:
+ generator = new SequentialDateDataGenerator(column);
+ break;
+ case BIGINT:
+ case INTEGER:
+ case TINYINT:
+ case UNSIGNED_LONG:
+ generator = new SequentialIntegerDataGenerator(column);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ String.format("No rule based generator supported for column type %s on %s",
+ column.getType(), column.getName()));
+ }
+ RuleBasedDataGenerator oldGenerator = columnRuleBasedDataGeneratorMap.putIfAbsent(column.getName(),generator);
+ if (oldGenerator != null) {
+ // Another thread succeeded in registering their generator first, so let's use that.
+ generator = oldGenerator;
+ }
}
return generator;
}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java
new file mode 100644
index 0000000..40deadd
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialDateDataGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.rules;
+
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+
+import org.joda.time.LocalDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A generator for sequentially increasing dates.
+ * For now the increments are fixed at 1 second.
+ */
+public class SequentialDateDataGenerator implements RuleBasedDataGenerator {
+ private static DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+ private final Column columnRule;
+ private final AtomicInteger counter;
+ private final LocalDateTime startDateTime = new LocalDateTime();
+
+ public SequentialDateDataGenerator(Column columnRule) {
+ Preconditions.checkArgument(columnRule.getDataSequence() == DataSequence.SEQUENTIAL);
+ Preconditions.checkArgument(isDateType(columnRule.getType()));
+ this.columnRule = columnRule;
+ counter = new AtomicInteger(0);
+ }
+
+ /**
+ * Note that this method rolls over for attempts to get larger than maxValue
+ * @return new DataValue
+ */
+ @Override
+ public DataValue getDataValue() {
+ LocalDateTime newDateTime = startDateTime.plusSeconds(counter.getAndIncrement());
+ String formattedDateTime = newDateTime.toString(FMT);
+ return new DataValue(columnRule.getType(), formattedDateTime);
+ }
+
+ boolean isDateType(DataTypeMapping mapping) {
+ switch (mapping) {
+ case DATE:
+ case TIMESTAMP:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java
new file mode 100644
index 0000000..a828641
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialListDataGenerator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.rules;
+
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A generator to round robin thru a list of values.
+ */
+
+public class SequentialListDataGenerator implements RuleBasedDataGenerator {
+ private final Column columnRule;
+ private final AtomicLong counter;
+
+ public SequentialListDataGenerator(Column columnRule) {
+ Preconditions.checkArgument(columnRule.getDataSequence() == DataSequence.SEQUENTIAL);
+ Preconditions.checkArgument(columnRule.getDataValues().size() > 0);
+ Preconditions.checkArgument(isAllowedType(columnRule.getType()));
+ this.columnRule = columnRule;
+ counter = new AtomicLong(0);
+ }
+
+ /**
+ * Note that this method rolls over for attempts to get larger than maxValue
+ * @return new DataValue
+ */
+ @Override
+ public DataValue getDataValue() {
+ long pos = counter.getAndIncrement();
+ int index = (int) pos % columnRule.getDataValues().size();
+ return columnRule.getDataValues().get(index);
+ }
+
+ boolean isAllowedType(DataTypeMapping mapping) {
+ // For now only varchar list are supported
+ switch (mapping) {
+ case VARCHAR:
+ case VARBINARY:
+ case CHAR:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java
new file mode 100644
index 0000000..d334537
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGenerator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.rules;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A generator for sequentially increasing varchar values.
+ */
+public class SequentialVarcharDataGenerator implements RuleBasedDataGenerator {
+ private final Column columnRule;
+ private final AtomicLong counter;
+
+ public SequentialVarcharDataGenerator(Column columnRule) {
+ Preconditions.checkArgument(columnRule.getDataSequence() == DataSequence.SEQUENTIAL);
+ Preconditions.checkArgument(isVarcharType(columnRule.getType()));
+ this.columnRule = columnRule;
+ counter = new AtomicLong(0);
+ }
+
+ /**
+ * Add a numerically increasing counter onto the and of a random string.
+ * Incremented counter should be thread safe.
+ *
+ * @return {@link org.apache.phoenix.pherf.rules.DataValue}
+ */
+ @Override
+ public DataValue getDataValue() {
+ DataValue data = null;
+ long inc = counter.getAndIncrement();
+ String strInc = String.valueOf(inc);
+ int paddedLength = columnRule.getLengthExcludingPrefix();
+ String strInc1 = StringUtils.leftPad(strInc, paddedLength, "x");
+ String strInc2 = StringUtils.right(strInc1, columnRule.getLengthExcludingPrefix());
+ String varchar = (columnRule.getPrefix() != null) ? columnRule.getPrefix() + strInc2:
+ strInc2;
+
+ // Truncate string back down if it exceeds length
+ varchar = StringUtils.left(varchar,columnRule.getLength());
+ data = new DataValue(columnRule.getType(), varchar);
+ return data;
+ }
+
+ boolean isVarcharType(DataTypeMapping mapping) {
+ switch (mapping) {
+ case VARCHAR:
+ case VARBINARY:
+ case CHAR:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 34f45b2..fd53b97 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -18,16 +18,37 @@
package org.apache.phoenix.pherf.util;
+import com.google.gson.Gson;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.mapreduce.index.automation.PhoenixMRJobSubmitter;
import org.apache.phoenix.pherf.PherfConstants;
-import org.apache.phoenix.pherf.configuration.*;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+import org.apache.phoenix.pherf.configuration.Ddl;
+import org.apache.phoenix.pherf.configuration.Query;
+import org.apache.phoenix.pherf.configuration.QuerySet;
+import org.apache.phoenix.pherf.configuration.Scenario;
import org.apache.phoenix.pherf.result.DataLoadTimeSummary;
+import org.apache.phoenix.pherf.rules.DataValue;
import org.apache.phoenix.pherf.rules.RulesApplier;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.sql.*;
+import java.math.BigDecimal;
+import java.sql.Array;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -40,6 +61,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
public class PhoenixUtil {
+ public static final String ASYNC_KEYWORD = "ASYNC";
+ public static final Gson GSON = new Gson();
private static final Logger LOGGER = LoggerFactory.getLogger(PhoenixUtil.class);
private static String zookeeper;
private static int rowCountOverride = 0;
@@ -47,7 +70,6 @@ public class PhoenixUtil {
private static PhoenixUtil instance;
private static boolean useThinDriver;
private static String queryServerUrl;
- private static final String ASYNC_KEYWORD = "ASYNC";
private static final int ONE_MIN_IN_MS = 60000;
private static String CurrentSCN = null;
@@ -81,6 +103,10 @@ public class PhoenixUtil {
return PhoenixUtil.useThinDriver;
}
+ public static Gson getGSON() {
+ return GSON;
+ }
+
public Connection getConnection() throws Exception {
return getConnection(null);
}
@@ -236,6 +262,15 @@ public class PhoenixUtil {
}
}
+ public void dropChildView(RegionCoprocessorEnvironment taskRegionEnvironment, int depth) {
+ TaskRegionObserver.SelfHealingTask task =
+ new TaskRegionObserver.SelfHealingTask(
+ taskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
+ for (int i = 0; i < depth; i++) {
+ task.run();
+ }
+ }
+
public ResultSet getTableMetaData(String schemaName, String tableName, Connection connection)
throws SQLException {
DatabaseMetaData dbmd = connection.getMetaData();
@@ -262,6 +297,7 @@ public class PhoenixUtil {
column.setType(DataTypeMapping.valueOf(resultSet.getString("TYPE_NAME").replace(" ", "_")));
column.setLength(resultSet.getInt("COLUMN_SIZE"));
columnList.add(column);
+ LOGGER.debug(String.format("getColumnsMetaData for column name : %s", column.getName()));
}
} finally {
if (null != resultSet) {
@@ -330,7 +366,7 @@ public class PhoenixUtil {
* @param tableName
* @throws InterruptedException
*/
- private void waitForAsyncIndexToFinish(String tableName) throws InterruptedException {
+ public void waitForAsyncIndexToFinish(String tableName) throws InterruptedException {
//Wait for up to 15 mins for ASYNC index build to start
boolean jobStarted = false;
for (int i=0; i<15; i++) {
@@ -450,4 +486,156 @@ public class PhoenixUtil {
}
return buf.toString();
}
+
+ public PreparedStatement buildStatement(RulesApplier rulesApplier, Scenario scenario, List<Column> columns,
+ PreparedStatement statement, SimpleDateFormat simpleDateFormat) throws Exception {
+
+ int count = 1;
+ for (Column column : columns) {
+ DataValue dataValue = rulesApplier.getDataForRule(scenario, column);
+ switch (column.getType()) {
+ case VARCHAR:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.VARCHAR);
+ } else {
+ statement.setString(count, dataValue.getValue());
+ }
+ break;
+ case CHAR:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.CHAR);
+ } else {
+ statement.setString(count, dataValue.getValue());
+ }
+ break;
+ case DECIMAL:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.DECIMAL);
+ } else {
+ statement.setBigDecimal(count, new BigDecimal(dataValue.getValue()));
+ }
+ break;
+ case INTEGER:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.INTEGER);
+ } else {
+ statement.setInt(count, Integer.parseInt(dataValue.getValue()));
+ }
+ break;
+ case UNSIGNED_LONG:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.OTHER);
+ } else {
+ statement.setLong(count, Long.parseLong(dataValue.getValue()));
+ }
+ break;
+ case BIGINT:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.BIGINT);
+ } else {
+ statement.setLong(count, Long.parseLong(dataValue.getValue()));
+ }
+ break;
+ case TINYINT:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.TINYINT);
+ } else {
+ statement.setLong(count, Integer.parseInt(dataValue.getValue()));
+ }
+ break;
+ case DATE:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.DATE);
+ } else {
+ Date
+ date =
+ new java.sql.Date(simpleDateFormat.parse(dataValue.getValue()).getTime());
+ statement.setDate(count, date);
+ }
+ break;
+ case VARCHAR_ARRAY:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.ARRAY);
+ } else {
+ Array
+ arr =
+ statement.getConnection().createArrayOf("VARCHAR", dataValue.getValue().split(","));
+ statement.setArray(count, arr);
+ }
+ break;
+ case VARBINARY:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.VARBINARY);
+ } else {
+ statement.setBytes(count, dataValue.getValue().getBytes());
+ }
+ break;
+ case TIMESTAMP:
+ if (dataValue.getValue().equals("")) {
+ statement.setNull(count, Types.TIMESTAMP);
+ } else {
+ java.sql.Timestamp
+ ts =
+ new java.sql.Timestamp(simpleDateFormat.parse(dataValue.getValue()).getTime());
+ statement.setTimestamp(count, ts);
+ }
+ break;
+ default:
+ break;
+ }
+ count++;
+ }
+ return statement;
+ }
+
+ public String buildSql(final List<Column> columns, final String tableName) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("upsert into ");
+ builder.append(tableName);
+ builder.append(" (");
+ int count = 1;
+ for (Column column : columns) {
+ builder.append(column.getName());
+ if (count < columns.size()) {
+ builder.append(",");
+ } else {
+ builder.append(")");
+ }
+ count++;
+ }
+ builder.append(" VALUES (");
+ for (int i = 0; i < columns.size(); i++) {
+ if (i < columns.size() - 1) {
+ builder.append("?,");
+ } else {
+ builder.append("?)");
+ }
+ }
+ return builder.toString();
+ }
+
+ public org.apache.hadoop.hbase.util.Pair<Long, Long> getResults(
+ Query query,
+ ResultSet rs,
+ String queryIteration,
+ boolean isSelectCountStatement,
+ Long queryStartTime) throws Exception {
+
+ Long resultRowCount = 0L;
+ while (rs.next()) {
+ if (isSelectCountStatement) {
+ resultRowCount = rs.getLong(1);
+ } else {
+ resultRowCount++;
+ }
+ long queryElapsedTime = EnvironmentEdgeManager.currentTimeMillis() - queryStartTime;
+ if (queryElapsedTime >= query.getTimeoutDuration()) {
+ LOGGER.error("Query " + queryIteration + " exceeded timeout of "
+ + query.getTimeoutDuration() + " ms at " + queryElapsedTime + " ms.");
+ return new org.apache.hadoop.hbase.util.Pair(resultRowCount, queryElapsedTime);
+ }
+ }
+ return new org.apache.hadoop.hbase.util.Pair(resultRowCount, EnvironmentEdgeManager.currentTimeMillis() - queryStartTime);
+ }
+
}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
index 64ee6ee..d3942c4 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/ResourceList.java
@@ -29,7 +29,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
@@ -76,7 +78,7 @@ public class ResourceList {
final String classPath = System.getProperty("java.class.path", ".");
final String[] classPathElements = classPath.split(":");
- List<String> strResources = new ArrayList<>();
+ Set<String> strResources = new HashSet<>();
Collection<Path> paths = new ArrayList<>();
// TODO Make getResourcesPaths() return the URLs directly instead of converting them
@@ -112,6 +114,7 @@ public class ResourceList {
paths.add(path);
}
+ Collections.sort((List<Path>)paths);
return paths;
}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java
index c4c38bd..bed2735 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/MultiThreadedRunner.java
@@ -161,7 +161,7 @@ class MultiThreadedRunner implements Callable<Void> {
conn.setAutoCommit(true);
final String statementString = query.getDynamicStatement(ruleApplier, scenario);
statement = conn.prepareStatement(statementString);
- LOGGER.info("Executing iteration: " + queryIteration + ": " + statementString);
+ LOGGER.debug("Executing iteration: " + queryIteration + ": " + statementString);
if (scenario.getWriteParams() != null) {
Workload writes = new WriteWorkload(PhoenixUtil.create(), parser, scenario, GeneratePhoenixStats.NO);
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index 613fb23..b6a5ac6 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -274,11 +274,11 @@ public class WriteWorkload implements Workload {
logPerNRows = Integer.valueOf(customizedLogPerNRows);
}
last = start = EnvironmentEdgeManager.currentTimeMillis();
- String sql = buildSql(columns, tableName);
+ String sql = pUtil.buildSql(columns, tableName);
stmt = connection.prepareStatement(sql);
for (long i = rowCount; (i > 0) && ((EnvironmentEdgeManager.currentTimeMillis() - logStartTime)
< maxDuration); i--) {
- stmt = buildStatement(scenario, columns, stmt, simpleDateFormat);
+ stmt = pUtil.buildStatement(rulesApplier, scenario, columns, stmt, simpleDateFormat);
if (useBatchApi) {
stmt.addBatch();
} else {
@@ -362,133 +362,6 @@ public class WriteWorkload implements Workload {
return future;
}
- private PreparedStatement buildStatement(Scenario scenario, List<Column> columns,
- PreparedStatement statement, SimpleDateFormat simpleDateFormat) throws Exception {
- int count = 1;
- for (Column column : columns) {
-
- DataValue dataValue = getRulesApplier().getDataForRule(scenario, column);
- switch (column.getType()) {
- case VARCHAR:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.VARCHAR);
- } else {
- statement.setString(count, dataValue.getValue());
- }
- break;
- case CHAR:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.CHAR);
- } else {
- statement.setString(count, dataValue.getValue());
- }
- break;
- case DECIMAL:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.DECIMAL);
- } else {
- statement.setBigDecimal(count, new BigDecimal(dataValue.getValue()));
- }
- break;
- case INTEGER:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.INTEGER);
- } else {
- statement.setInt(count, Integer.parseInt(dataValue.getValue()));
- }
- break;
- case UNSIGNED_LONG:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.OTHER);
- } else {
- statement.setLong(count, Long.parseLong(dataValue.getValue()));
- }
- break;
- case BIGINT:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.BIGINT);
- } else {
- statement.setLong(count, Long.parseLong(dataValue.getValue()));
- }
- break;
- case TINYINT:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.TINYINT);
- } else {
- statement.setLong(count, Integer.parseInt(dataValue.getValue()));
- }
- break;
- case DATE:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.DATE);
- } else {
- Date
- date =
- new java.sql.Date(simpleDateFormat.parse(dataValue.getValue()).getTime());
- statement.setDate(count, date);
- }
- break;
- case VARCHAR_ARRAY:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.ARRAY);
- } else {
- Array
- arr =
- statement.getConnection().createArrayOf("VARCHAR", dataValue.getValue().split(","));
- statement.setArray(count, arr);
- }
- break;
- case VARBINARY:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.VARBINARY);
- } else {
- statement.setBytes(count, dataValue.getValue().getBytes());
- }
- break;
- case TIMESTAMP:
- if (dataValue.getValue().equals("")) {
- statement.setNull(count, Types.TIMESTAMP);
- } else {
- java.sql.Timestamp
- ts =
- new java.sql.Timestamp(simpleDateFormat.parse(dataValue.getValue()).getTime());
- statement.setTimestamp(count, ts);
- }
- break;
- default:
- break;
- }
- count++;
- }
- return statement;
- }
-
- private String buildSql(final List<Column> columns, final String tableName) {
- StringBuilder builder = new StringBuilder();
- builder.append("upsert into ");
- builder.append(tableName);
- builder.append(" (");
- int count = 1;
- for (Column column : columns) {
- builder.append(column.getName());
- if (count < columns.size()) {
- builder.append(",");
- } else {
- builder.append(")");
- }
- count++;
- }
- builder.append(" VALUES (");
- for (int i = 0; i < columns.size(); i++) {
- if (i < columns.size() - 1) {
- builder.append("?,");
- } else {
- builder.append("?)");
- }
- }
- return builder.toString();
- }
-
public XMLConfigParser getParser() {
return parser;
}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/MultiTenantWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/MultiTenantWorkload.java
new file mode 100644
index 0000000..c8fa699
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/MultiTenantWorkload.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.Workload;
+import org.apache.phoenix.pherf.workload.mt.generators.LoadEventGenerator;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantLoadEventGeneratorFactory;
+import org.apache.phoenix.pherf.workload.mt.handlers.TenantOperationWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+
+/**
+ * This class creates workload for tenant based load profiles.
+ * It uses @see {@link TenantOperationFactory} in conjunction with
+ * @see {@link LoadEventGenerator} to generate the load events.
+ * It then publishes these events onto a RingBuffer based queue.
+ * The @see {@link TenantOperationWorkHandler} drains the events from the queue and executes them.
+ * Reference for RingBuffer based queue http://lmax-exchange.github.io/disruptor/
+ */
+
+public class MultiTenantWorkload implements Workload {
+ private static final Logger LOGGER = LoggerFactory.getLogger(MultiTenantWorkload.class);
+ private final TenantLoadEventGeneratorFactory evtGeneratorFactory
+ = new TenantLoadEventGeneratorFactory();
+ private final LoadEventGenerator<TenantOperationInfo> generator;
+
+
+ public MultiTenantWorkload(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ Properties properties) {
+ this.generator = evtGeneratorFactory.newLoadEventGenerator(phoenixUtil,
+ model, scenario, properties);
+ }
+
+ public MultiTenantWorkload(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workHandlers, Properties properties) throws Exception {
+ this.generator = evtGeneratorFactory.newLoadEventGenerator(phoenixUtil,
+ model, scenario, workHandlers, properties);
+ }
+
+ @Override public Callable<Void> execute() throws Exception {
+ return new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ generator.start();
+ return null;
+ }
+ };
+ }
+
+ @Override public void complete() {
+ try {
+ generator.stop();
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
+ }
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/BaseLoadEventGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/BaseLoadEventGenerator.java
new file mode 100644
index 0000000..dbb4c08
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/BaseLoadEventGenerator.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import com.lmax.disruptor.BlockingWaitStrategy;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.ExceptionHandler;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.WorkHandler;
+import com.lmax.disruptor.dsl.Disruptor;
+import com.lmax.disruptor.dsl.ProducerType;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+import org.apache.phoenix.pherf.workload.mt.handlers.TenantOperationWorkHandler;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Properties;
+/**
+ * A base class for all load event generators.
+ */
+public abstract class BaseLoadEventGenerator
+ implements LoadEventGenerator<TenantOperationInfo> {
+ protected static final int DEFAULT_NUM_HANDLER_PER_SCENARIO = 4;
+ protected static final int DEFAULT_BUFFER_SIZE = 8192;
+ protected static final Logger LOGGER = LoggerFactory.getLogger(
+ BaseLoadEventGenerator.class);
+
+ protected Disruptor<TenantOperationEvent> disruptor;
+ protected List<PherfWorkHandler> handlers;
+ protected final Properties properties;
+
+ protected final TenantOperationFactory operationFactory;
+ protected final ExceptionHandler exceptionHandler;
+
+
+ private static class WorkloadExceptionHandler implements ExceptionHandler {
+ private static final Logger LOGGER = LoggerFactory.getLogger(WorkloadExceptionHandler.class);
+
+ @Override public void handleEventException(Throwable ex, long sequence, Object event) {
+ LOGGER.error("Sequence=" + sequence + ", event=" + event, ex);
+ throw new RuntimeException(ex);
+ }
+
+ @Override public void handleOnStartException(Throwable ex) {
+ LOGGER.error("On Start", ex);
+ throw new RuntimeException(ex);
+ }
+
+ @Override public void handleOnShutdownException(Throwable ex) {
+ LOGGER.error("On Shutdown", ex);
+ throw new RuntimeException(ex);
+ }
+ }
+
+ public static class TenantOperationEvent {
+ TenantOperationInfo tenantOperationInfo;
+
+ public TenantOperationInfo getTenantOperationInfo() {
+ return tenantOperationInfo;
+ }
+
+ public void setTenantOperationInfo(TenantOperationInfo tenantOperationInfo) {
+ this.tenantOperationInfo = tenantOperationInfo;
+ }
+
+ public static final EventFactory<TenantOperationEvent> EVENT_FACTORY = new EventFactory<TenantOperationEvent>() {
+ public TenantOperationEvent newInstance() {
+ return new TenantOperationEvent();
+ }
+ };
+ }
+
+ public BaseLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workers, Properties properties) {
+ this(phoenixUtil, model, scenario, workers, new WorkloadExceptionHandler(), properties);
+ }
+
+ public BaseLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model,
+ Scenario scenario, Properties properties) {
+ this(phoenixUtil, model, scenario, null, new WorkloadExceptionHandler(), properties);
+ }
+
+ public BaseLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model,
+ Scenario scenario,
+ List<PherfWorkHandler> workers,
+ ExceptionHandler exceptionHandler,
+ Properties properties) {
+
+
+ operationFactory = new TenantOperationFactory(phoenixUtil, model, scenario);
+ if (scenario.getPhoenixProperties() != null) {
+ properties.putAll(scenario.getPhoenixProperties());
+ }
+ this.properties = properties;
+
+ if (workers == null || workers.isEmpty()) {
+ workers = getWorkHandlers(properties);
+ }
+ this.handlers = workers;
+ this.exceptionHandler = exceptionHandler;
+ }
+
+
+ @Override public PhoenixUtil getPhoenixUtil() { return operationFactory.getPhoenixUtil(); }
+
+ @Override public Scenario getScenario() {
+ return operationFactory.getScenario();
+ }
+
+ @Override public DataModel getModel() {
+ return operationFactory.getModel();
+ }
+
+ @Override public Properties getProperties() {
+ return this.properties;
+ }
+
+ @Override public TenantOperationFactory getOperationFactory() {
+ return operationFactory;
+ }
+
+ @Override public void start() throws Exception {
+ Scenario scenario = operationFactory.getScenario();
+ String currentThreadName = Thread.currentThread().getName();
+ int bufferSize = DEFAULT_BUFFER_SIZE;
+ if (properties.containsKey("pherf.mt.buffer_size_per_scenario")) {
+ bufferSize = Integer.parseInt((String)properties.get("pherf.mt.buffer_size_per_scenario"));
+ }
+
+ disruptor = new Disruptor<TenantOperationEvent>(TenantOperationEvent.EVENT_FACTORY, bufferSize,
+ new ThreadFactoryBuilder()
+ .setNameFormat(currentThreadName + "." + scenario.getName())
+ .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER)
+ .build(),
+ ProducerType.SINGLE, new BlockingWaitStrategy());
+
+ this.disruptor.setDefaultExceptionHandler(this.exceptionHandler);
+ this.disruptor.handleEventsWithWorkerPool(this.handlers.toArray(new WorkHandler[] {}));
+ RingBuffer<TenantOperationEvent> ringBuffer = this.disruptor.start();
+ long numOperations = scenario.getLoadProfile().getNumOperations();
+ while (numOperations > 0) {
+ TenantOperationInfo sample = next();
+ operationFactory.initializeTenant(sample);
+ --numOperations;
+ // Publishers claim events in sequence
+ long sequence = ringBuffer.next();
+ TenantOperationEvent event = ringBuffer.get(sequence);
+ event.setTenantOperationInfo(sample);
+ // make the event available to EventProcessors
+ ringBuffer.publish(sequence);
+ LOGGER.info(String.format("published : %s:%s:%d, %d, %d",
+ scenario.getName(), scenario.getTableName(),
+ numOperations, ringBuffer.getCursor(), sequence));
+ }
+ }
+
+ @Override public void stop() throws Exception {
+ // Wait for the handlers to finish the jobs
+ if (disruptor != null) {
+ disruptor.shutdown();
+ }
+
+ // TODO need to handle asynchronous result publishing
+ }
+
+ @Override public List<PherfWorkHandler> getWorkHandlers(Properties properties) {
+
+ String handlerName = "";
+ try {
+ handlerName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ throw new RuntimeException(e);
+ }
+
+ int handlerCount = DEFAULT_NUM_HANDLER_PER_SCENARIO;
+ if (properties.containsKey("pherf.mt.handlers_per_scenario")) {
+ handlerCount = Integer.parseInt((String)properties.get("pherf.mt.handlers_per_scenario"));
+ }
+ List<PherfWorkHandler> workers = Lists.newArrayListWithCapacity(handlerCount);
+ for (int i = 0; i < handlerCount; i++) {
+ String handlerId = String.format("%s.%d", handlerName, i + 1);
+ workers.add(new TenantOperationWorkHandler(operationFactory, handlerId));
+ }
+ return workers;
+ }
+
+ abstract public TenantOperationInfo next();
+
+
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/LoadEventGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/LoadEventGenerator.java
new file mode 100644
index 0000000..61f8fe7
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/LoadEventGenerator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * An interface that implementers can use to generate load events that can be consumed by
+ * @see {@link com.lmax.disruptor.WorkHandler} which provide event handling functionality for
+ * a given event.
+ *
+ * @param <T> load event object
+ */
+public interface LoadEventGenerator<T> {
+ /**
+ * Initializes and readies the generator for queue based workloads
+ */
+ void start() throws Exception;
+
+ /**
+ * Stop the generator and waits for the queues to drain.
+ */
+ void stop() throws Exception;
+
+ PhoenixUtil getPhoenixUtil();
+
+ Scenario getScenario();
+
+ DataModel getModel();
+
+ Properties getProperties();
+
+ TenantOperationFactory getOperationFactory();
+
+ List<PherfWorkHandler> getWorkHandlers(Properties properties);
+
+ TenantOperationInfo next();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/LoadEventGeneratorFactory.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/LoadEventGeneratorFactory.java
new file mode 100644
index 0000000..63e482c
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/LoadEventGeneratorFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * An interface that factory implementers need to implement
+ * for creating various supported load generators {@link LoadEventGenerator}
+ * @param <T> load event object
+ */
+public interface LoadEventGeneratorFactory<T> {
+ LoadEventGenerator<T> newLoadEventGenerator(PhoenixUtil phoenixUtil,
+ DataModel model, Scenario scenario,
+ Properties properties) ;
+
+ LoadEventGenerator<T> newLoadEventGenerator(PhoenixUtil phoenixUtil,
+ DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workHandlers, Properties properties) ;
+
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/SequentialLoadEventGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/SequentialLoadEventGenerator.java
new file mode 100644
index 0000000..cfcf497
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/SequentialLoadEventGenerator.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.ExecutionType;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.handlers.RendezvousingWorkHandler;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.CyclicBarrier;
+
+/**
+ * A load generator that generates tenant operation events in the order specified in the
+ * scenario file.
+ * The scenario file can also specify on how many iterations to be executed per operation,
+ * whether the iterations be run in parallel or serially.
+ */
+public class SequentialLoadEventGenerator extends BaseLoadEventGenerator {
+
+ private static class SequentialSampler {
+ private final LoadProfile loadProfile;
+ private final String modelName;
+ private final String scenarioName;
+ private final String tableName;
+ private long iteration;
+ private int opIndex;
+ private int numHandlers;
+
+ private final TenantGroup tenantGroup;
+ private final List<Operation> operationList;
+
+ public SequentialSampler(List<Operation> operationList, DataModel model,
+ Scenario scenario, Properties properties) {
+ this.modelName = model.getName();
+ this.scenarioName = scenario.getName();
+ this.tableName = scenario.getTableName();
+ this.loadProfile = scenario.getLoadProfile();
+ this.operationList = operationList;
+
+ // Track the individual tenant group with single tenant or global connection,
+ // so that given a generated sample we can use the supplied tenant.
+ // NOTE : Not sure if there is a case for multiple tenants in a uniform distribution.
+ // For now keeping it simple.
+ Preconditions.checkArgument(loadProfile.getTenantDistribution() != null,
+ "Tenant distribution cannot be null");
+ Preconditions.checkArgument(!loadProfile.getTenantDistribution().isEmpty(),
+ "Tenant group cannot be empty");
+ Preconditions.checkArgument(loadProfile.getTenantDistribution().size() == 1,
+ "Tenant group cannot be more than 1");
+ tenantGroup = loadProfile.getTenantDistribution().get(0);
+ }
+
+ public TenantOperationInfo nextSample() {
+ Operation op = operationList.get(opIndex % operationList.size());
+ String tenantGroupId = tenantGroup.getId();
+ String tenantIdPrefix = Strings
+ .padStart(tenantGroupId, loadProfile.getGroupIdLength(), 'x');
+ String formattedTenantId = String.format(loadProfile.getTenantIdFormat(),
+ tenantIdPrefix.substring(0, loadProfile.getGroupIdLength()), 1);
+ String paddedTenantId = Strings.padStart(formattedTenantId, loadProfile.getTenantIdLength(), 'x');
+ String tenantId = paddedTenantId.substring(0, loadProfile.getTenantIdLength());
+
+ TenantOperationInfo sample = new TenantOperationInfo(modelName, scenarioName, tableName,
+ tenantGroupId, op.getId(), tenantId, op);
+
+ iteration++;
+ if (iteration % numHandlers == 0) {
+ opIndex++;
+ }
+ return sample;
+ }
+
+ public int getNumHandlers() {
+ return numHandlers;
+ }
+
+ public void setNumHandlers(int handlers) {
+ numHandlers = handlers;
+ }
+
+ }
+
+ protected static final int DEFAULT_NUM_ITERATIONS = 1;
+ protected static final ExecutionType DEFAULT_EXECUTION_TYPE = ExecutionType.SERIAL;
+ private final SequentialSampler sampler;
+ private int numHandlers;
+ private int numIterations = DEFAULT_NUM_ITERATIONS;
+ private ExecutionType executionType = DEFAULT_EXECUTION_TYPE;
+
+ public SequentialLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ Properties properties) {
+ super(phoenixUtil, model, scenario, properties);
+ this.sampler = new SequentialSampler(operationFactory.getOperations(), model, scenario, properties);
+ this.sampler.setNumHandlers(this.numHandlers);
+ }
+
+ public SequentialLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workHandlers, Properties properties) {
+ super(phoenixUtil, model, scenario, workHandlers, properties);
+ this.sampler = new SequentialSampler(operationFactory.getOperations(), model, scenario, properties);
+ this.sampler.setNumHandlers(this.numHandlers);
+ }
+
+ public List<PherfWorkHandler> getWorkHandlers(Properties properties) {
+
+ String handlerName = "";
+ try {
+ handlerName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ throw new RuntimeException(e);
+ }
+
+ this.numHandlers = DEFAULT_NUM_HANDLER_PER_SCENARIO;
+ if (properties.containsKey(PherfConstants.HANDLERS_PER_SCENARIO_PROP_KEY)) {
+ this.numHandlers = Integer.parseInt((String)properties.get(PherfConstants.HANDLERS_PER_SCENARIO_PROP_KEY));
+ }
+
+ if (properties.containsKey(PherfConstants.NUM_SEQUENTIAL_ITERATIONS_PROP_KEY)) {
+ this.numIterations = Integer.parseInt((String)properties.get(PherfConstants.NUM_SEQUENTIAL_ITERATIONS_PROP_KEY));
+ }
+
+ if (properties.containsKey(PherfConstants.NUM_SEQUENTIAL_EXECUTION_TYPE_PROP_KEY)) {
+ this.executionType = ExecutionType.valueOf((String)properties.get(PherfConstants.NUM_SEQUENTIAL_EXECUTION_TYPE_PROP_KEY));
+ switch (executionType) {
+ case SERIAL:
+ this.numHandlers = DEFAULT_NUM_ITERATIONS;
+ break;
+ case PARALLEL:
+ this.numHandlers = numIterations;
+ break;
+ default:
+ // Just accepts the defaults, nothing to do here
+ }
+ }
+
+ Map<String, CyclicBarrier> rendezvousPoints = Maps.newHashMap();
+ CyclicBarrier startBarrier = new CyclicBarrier(numHandlers, new Runnable() {
+ @Override public void run() {
+ LOGGER.info("Rendezvoused for start of operation execution");
+ }
+ });
+ rendezvousPoints.put(PherfConstants.MT_HANDLER_START_RENDEZVOUS_PROP_KEY, startBarrier);
+
+ List<PherfWorkHandler> workers = Lists.newArrayListWithCapacity(numHandlers);
+ for (int i = 0; i < numHandlers; i++) {
+ String handlerId = String.format("%s.%d", handlerName, i + 1);
+ workers.add(new RendezvousingWorkHandler(operationFactory, handlerId,
+ rendezvousPoints));
+ }
+ return workers;
+ }
+
+ @Override public TenantOperationInfo next() {
+ return this.sampler.nextSample();
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/TenantLoadEventGeneratorFactory.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/TenantLoadEventGeneratorFactory.java
new file mode 100644
index 0000000..31c1033
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/TenantLoadEventGeneratorFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * A factory class for creating various supported load generators {@link LoadEventGenerator}
+ */
+public class TenantLoadEventGeneratorFactory implements
+ LoadEventGeneratorFactory<TenantOperationInfo> {
+ public enum GeneratorType {
+ WEIGHTED, UNIFORM, SEQUENTIAL
+ }
+ @Override public LoadEventGenerator<TenantOperationInfo> newLoadEventGenerator(PhoenixUtil phoenixUtil,
+ DataModel model, Scenario scenario,
+ Properties properties) {
+ GeneratorType type = GeneratorType.valueOf(scenario.getGeneratorName());
+ switch (type) {
+ case WEIGHTED:
+ return new WeightedRandomLoadEventGenerator(phoenixUtil, model, scenario, properties);
+ case UNIFORM:
+ return new UniformDistributionLoadEventGenerator(phoenixUtil, model, scenario, properties);
+ case SEQUENTIAL:
+ return new SequentialLoadEventGenerator(phoenixUtil, model, scenario, properties);
+ default:
+ throw new IllegalArgumentException("Unknown generator type");
+ }
+ }
+
+ @Override public LoadEventGenerator<TenantOperationInfo> newLoadEventGenerator(PhoenixUtil phoenixUtil,
+ DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workHandlers, Properties properties) {
+ GeneratorType type = GeneratorType.valueOf(scenario.getGeneratorName());
+ switch (type) {
+ case WEIGHTED:
+ return new WeightedRandomLoadEventGenerator(phoenixUtil, model, scenario, workHandlers, properties);
+ case UNIFORM:
+ return new UniformDistributionLoadEventGenerator(phoenixUtil, model, scenario, workHandlers, properties);
+ case SEQUENTIAL:
+ return new SequentialLoadEventGenerator(phoenixUtil, model, scenario, workHandlers, properties);
+ default:
+ throw new IllegalArgumentException("Unknown generator type");
+ }
+
+ }
+
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/TenantOperationInfo.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/TenantOperationInfo.java
new file mode 100644
index 0000000..3b862f3
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/TenantOperationInfo.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+
+/**
+ * Holds information on the tenant operation details.
+ */
+public class TenantOperationInfo {
+ private final String modelName;
+ private final String scenarioName;
+ private final String tableName;
+ private final String tenantId;
+ private final String tenantGroupId;
+ private final String operationGroupId;
+ private final Operation operation;
+
+ public TenantOperationInfo(String modelName, String scenarioName, String tableName,
+ String tenantGroupId, String operationGroupId,
+ String tenantId, Operation operation) {
+ this.modelName = modelName;
+ this.scenarioName = scenarioName;
+ this.tableName = tableName;
+ this.tenantGroupId = tenantGroupId;
+ this.operationGroupId = operationGroupId;
+ this.tenantId = tenantId;
+ this.operation = operation;
+ }
+
+ public String getModelName() { return modelName; }
+
+ public String getScenarioName() { return scenarioName; }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getTenantGroupId() {
+ return tenantGroupId;
+ }
+
+ public String getOperationGroupId() {
+ return operationGroupId;
+ }
+
+ public Operation getOperation() {
+ return operation;
+ }
+
+ public String getTenantId() {
+ return tenantId;
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/UniformDistributionLoadEventGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/UniformDistributionLoadEventGenerator.java
new file mode 100644
index 0000000..2616672
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/UniformDistributionLoadEventGenerator.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+/**
+ * A load generator that generates a uniform distribution of operations among the given tenant group.
+ */
+public class UniformDistributionLoadEventGenerator extends BaseLoadEventGenerator {
+
+ private static class UniformDistributionSampler {
+ private final LoadProfile loadProfile;
+ private final String modelName;
+ private final String scenarioName;
+ private final String tableName;
+ private final Random distribution;
+
+ private final TenantGroup tenantGroup;
+ private final List<Operation> operationList;
+
+ public UniformDistributionSampler(List<Operation> operationList, DataModel model,
+ Scenario scenario) {
+ this.modelName = model.getName();
+ this.scenarioName = scenario.getName();
+ this.tableName = scenario.getTableName();
+ this.loadProfile = scenario.getLoadProfile();
+ this.operationList = operationList;
+
+ // Track the individual tenant group with single tenant or global connection,
+ // so that given a generated sample we can use the supplied tenant.
+ // NOTE : Not sure if there is a case for multiple tenants in a uniform distribution.
+ // For now keeping it simple.
+ Preconditions.checkArgument(loadProfile.getTenantDistribution() != null,
+ "Tenant distribution cannot be null");
+ Preconditions.checkArgument(!loadProfile.getTenantDistribution().isEmpty(),
+ "Tenant group cannot be empty");
+ Preconditions.checkArgument(loadProfile.getTenantDistribution().size() == 1,
+ "Tenant group cannot be more than 1");
+ tenantGroup = loadProfile.getTenantDistribution().get(0);
+
+ this.distribution = new Random();
+ }
+
+ public TenantOperationInfo nextSample() {
+ int sampleIndex = this.distribution.nextInt(operationList.size());
+ Operation op = operationList.get(sampleIndex);
+
+ String tenantGroupId = tenantGroup.getId();
+ String tenantIdPrefix = Strings
+ .padStart(tenantGroupId, loadProfile.getGroupIdLength(), 'x');
+ String formattedTenantId = String.format(loadProfile.getTenantIdFormat(),
+ tenantIdPrefix.substring(0, loadProfile.getGroupIdLength()), 1);
+ String paddedTenantId = Strings.padStart(formattedTenantId, loadProfile.getTenantIdLength(), 'x');
+ String tenantId = paddedTenantId.substring(0, loadProfile.getTenantIdLength());
+
+ TenantOperationInfo sample = new TenantOperationInfo(modelName, scenarioName, tableName,
+ tenantGroupId, op.getId(), tenantId, op);
+ return sample;
+ }
+ }
+
+ private final UniformDistributionSampler sampler;
+
+
+ public UniformDistributionLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ Properties properties) {
+ super(phoenixUtil, model, scenario, properties);
+ this.sampler = new UniformDistributionSampler(operationFactory.getOperations(), model, scenario);
+ }
+
+ public UniformDistributionLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workHandlers, Properties properties) {
+ super(phoenixUtil, model, scenario, workHandlers, properties);
+ this.sampler = new UniformDistributionSampler(operationFactory.getOperations(), model, scenario);
+ }
+
+
+ @Override public TenantOperationInfo next() {
+ return this.sampler.nextSample();
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/WeightedRandomLoadEventGenerator.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/WeightedRandomLoadEventGenerator.java
new file mode 100644
index 0000000..ebe8d64
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/generators/WeightedRandomLoadEventGenerator.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.generators;
+
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.handlers.PherfWorkHandler;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.math3.distribution.EnumeratedDistribution;
+import org.apache.commons.math3.util.Pair;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.OperationGroup;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+
+/**
+ * A perf load event generator based on the supplied load profile.
+ * The load profile enumerates the distribution of operation among the different tenant group
+ * which is used by the load generator to generate the operation events for the various tenants.
+ */
+
+public class WeightedRandomLoadEventGenerator extends BaseLoadEventGenerator {
+
+ private static class WeightedRandomSampler {
+ private static String AUTO_WEIGHTED_OPERATION_ID = "xxxxxx";
+ private final Random RANDOM = new Random();
+ private final LoadProfile loadProfile;
+ private final String modelName;
+ private final String scenarioName;
+ private final String tableName;
+ private final EnumeratedDistribution<String> distribution;
+
+ private final Map<String, TenantGroup> tenantGroupMap = Maps.newHashMap();
+ private final Map<String, Operation> operationMap = Maps.newHashMap();
+ private final List<String> autoWeightedOperations = Lists.newArrayList();
+ private final int numAutoWeightedOperations;
+
+ public WeightedRandomSampler(List<Operation> operationList, DataModel model, Scenario scenario) {
+ this.modelName = model.getName();
+ this.scenarioName = scenario.getName();
+ this.tableName = scenario.getTableName();
+ this.loadProfile = scenario.getLoadProfile();
+
+ // Track the individual tenant group sizes,
+ // so that given a generated sample we can get a random tenant for a group.
+ for (TenantGroup tg : loadProfile.getTenantDistribution()) {
+ tenantGroupMap.put(tg.getId(), tg);
+ }
+ Preconditions.checkArgument(!tenantGroupMap.isEmpty(),
+ "Tenant group cannot be empty");
+
+ for (Operation op : operationList) {
+ for (OperationGroup loadOp : loadProfile.getOpDistribution()) {
+ if (op.getId().compareTo(loadOp.getId()) == 0) {
+ operationMap.put(op.getId(), op);
+ }
+ }
+ }
+ Preconditions.checkArgument(!operationMap.isEmpty(),
+ "Operation list and load profile operation do not match");
+ this.distribution = initProbabilityDistribution(scenario.getLoadProfile());
+ this.numAutoWeightedOperations = autoWeightedOperations.size();
+
+ }
+
+ public TenantOperationInfo nextSample() {
+ String sampleIndex = this.distribution.sample();
+ String[] parts = sampleIndex.split(":");
+ String tenantGroupId = parts[0];
+ String opId = parts[1];
+
+ Operation op = operationMap.get(opId);
+ if (op == null && opId.compareTo(AUTO_WEIGHTED_OPERATION_ID) == 0) {
+ opId = autoWeightedOperations.get(RANDOM.nextInt(numAutoWeightedOperations));
+ op = operationMap.get(opId);
+ }
+ int numTenants = tenantGroupMap.get(tenantGroupId).getNumTenants();
+ String tenantIdPrefix = Strings.padStart(tenantGroupId, loadProfile.getGroupIdLength(), 'x');
+ String formattedTenantId = String.format(loadProfile.getTenantIdFormat(),
+ tenantIdPrefix.substring(0, loadProfile.getGroupIdLength()), RANDOM.nextInt(numTenants));
+ String paddedTenantId = Strings.padStart(formattedTenantId, loadProfile.getTenantIdLength(), 'x');
+ String tenantId = paddedTenantId.substring(0, loadProfile.getTenantIdLength());
+
+ TenantOperationInfo sample = new TenantOperationInfo(modelName, scenarioName, tableName,
+ tenantGroupId, opId, tenantId, op);
+ return sample;
+ }
+
+ private EnumeratedDistribution initProbabilityDistribution(LoadProfile loadProfile) {
+ double totalTenantGroupWeight = 0.0f;
+ double totalOperationWeight = 0.0f;
+ double remainingOperationWeight = 0.0f;
+
+ // Sum the weights to find the total weight,
+ // so that the weights can be used in the total probability distribution.
+ for (TenantGroup tg : loadProfile.getTenantDistribution()) {
+ Preconditions.checkArgument(tg.getWeight() > 0.0f,
+ "Tenant group weight cannot be less than zero");
+ totalTenantGroupWeight += tg.getWeight();
+ }
+ for (OperationGroup op : loadProfile.getOpDistribution()) {
+ if (op.getWeight() > 0.0f) {
+ totalOperationWeight += op.getWeight();
+ } else {
+ autoWeightedOperations.add(op.getId());
+ }
+ }
+
+ if (!autoWeightedOperations.isEmpty()) {
+ remainingOperationWeight = 100.0f - totalOperationWeight;
+ totalOperationWeight = 100.0f;
+ }
+
+ Preconditions.checkArgument(totalTenantGroupWeight == 100.0f,
+ "Total tenant group weight cannot be <> 100.0");
+ Preconditions.checkArgument(totalOperationWeight == 100.0f,
+ "Total operation group weight cannot be <> 100.0");
+
+ // Initialize the sample probability distribution
+ List<Pair<String, Double>> pmf = Lists.newArrayList();
+ double totalWeight = totalTenantGroupWeight * totalOperationWeight;
+ for (TenantGroup tg : loadProfile.getTenantDistribution()) {
+ for (OperationGroup op : loadProfile.getOpDistribution()) {
+ int opWeight = op.getWeight();
+ if (opWeight > 0.0f) {
+ String sampleName = String.format("%s:%s", tg.getId(), op.getId());
+ double probability = (tg.getWeight() * opWeight)/totalWeight;
+ pmf.add(new Pair(sampleName, probability));
+ }
+ }
+
+ if (!autoWeightedOperations.isEmpty()) {
+ String sampleName = String.format("%s:%s", tg.getId(), AUTO_WEIGHTED_OPERATION_ID);
+ double probability = (tg.getWeight() * remainingOperationWeight)/totalWeight;
+ pmf.add(new Pair(sampleName, probability));
+ }
+ }
+ EnumeratedDistribution distribution = new EnumeratedDistribution(pmf);
+ return distribution;
+ }
+ }
+
+ private final WeightedRandomSampler sampler;
+
+ public WeightedRandomLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ Properties properties) {
+ super(phoenixUtil, model, scenario, properties);
+ this.sampler = new WeightedRandomSampler(operationFactory.getOperations(), model, scenario);
+ }
+
+ public WeightedRandomLoadEventGenerator(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario,
+ List<PherfWorkHandler> workHandlers, Properties properties) {
+ super(phoenixUtil, model, scenario, workHandlers, properties);
+ this.sampler = new WeightedRandomSampler(operationFactory.getOperations(), model, scenario);
+ }
+
+ @Override public TenantOperationInfo next() {
+ return this.sampler.nextSample();
+ }
+
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/PherfWorkHandler.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/PherfWorkHandler.java
new file mode 100644
index 0000000..fc82b2e
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/PherfWorkHandler.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.handlers;
+
+import com.lmax.disruptor.WorkHandler;
+import org.apache.phoenix.pherf.result.ResultValue;
+import org.apache.phoenix.pherf.workload.mt.operations.OperationStats;
+
+import java.util.List;
+
+public interface PherfWorkHandler<T> extends WorkHandler<T> {
+ List<ResultValue<OperationStats>> getResults();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/RendezvousingWorkHandler.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/RendezvousingWorkHandler.java
new file mode 100644
index 0000000..6e4da93
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/RendezvousingWorkHandler.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.handlers;
+
+import com.lmax.disruptor.LifecycleAware;
+import com.lmax.disruptor.WorkHandler;
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.result.ResultValue;
+import org.apache.phoenix.pherf.workload.mt.MultiTenantWorkload;
+import org.apache.phoenix.pherf.workload.mt.generators.BaseLoadEventGenerator.TenantOperationEvent;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+import org.apache.phoenix.pherf.workload.mt.operations.OperationStats;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CyclicBarrier;
+
+/**
+ * A handler {@link WorkHandler} for simple orchestrations using the supplied rendezvous points
+ * The handler will wait for the rendezvous to happen before executing the operations {@link Operation}
+ * The handlers as in the basic {@link TenantOperationWorkHandler} will pick up the operation events
+ * as and when they become available on the {@link com.lmax.disruptor.RingBuffer}
+ * when published by the workload generator {@link MultiTenantWorkload}
+ */
+
+public class RendezvousingWorkHandler implements PherfWorkHandler<TenantOperationEvent>,
+ LifecycleAware {
+ private static final Logger LOGGER = LoggerFactory.getLogger(RendezvousingWorkHandler.class);
+ private final String handlerId;
+ private final TenantOperationFactory operationFactory;
+ private final Map<String, CyclicBarrier> rendezvousPoints;
+
+ public RendezvousingWorkHandler(TenantOperationFactory operationFactory,
+ String handlerId, Map<String, CyclicBarrier> rendezvousPoints) {
+ this.handlerId = handlerId;
+ this.operationFactory = operationFactory;
+ this.rendezvousPoints = rendezvousPoints;
+ }
+
+ @Override
+ public void onEvent(TenantOperationEvent event)
+ throws Exception {
+ TenantOperationInfo input = event.getTenantOperationInfo();
+ Supplier<Function<TenantOperationInfo, OperationStats>> opSupplier =
+ operationFactory.getOperationSupplier(input);
+
+ boolean startRendezvousEnabled = rendezvousPoints.containsKey(PherfConstants.MT_HANDLER_START_RENDEZVOUS_PROP_KEY);
+ if (startRendezvousEnabled) {
+ rendezvousPoints.get(PherfConstants.MT_HANDLER_START_RENDEZVOUS_PROP_KEY).await();
+ }
+ OperationStats stats = opSupplier.get().apply(input);
+ stats.setHandlerId(handlerId);
+
+ // TODO need to handle asynchronous result publishing
+ boolean resultsRendezvousEnabled = rendezvousPoints.containsKey(PherfConstants.MT_HANDLER_RESULTS_RENDEZVOUS_PROP_KEY);
+ if (resultsRendezvousEnabled) {
+ rendezvousPoints.get(PherfConstants.MT_HANDLER_RESULTS_RENDEZVOUS_PROP_KEY).await();
+ }
+ LOGGER.info(operationFactory.getPhoenixUtil().getGSON().toJson(stats));
+ }
+
+ @Override
+ public void onStart() {
+ Scenario scenario = operationFactory.getScenario();
+ LOGGER.info(String.format("TenantOperationWorkHandler started for %s:%s",
+ scenario.getName(), scenario.getTableName()));
+ }
+
+ @Override
+ public void onShutdown() {
+ Scenario scenario = operationFactory.getScenario();
+ LOGGER.info(String.format("TenantOperationWorkHandler stopped for %s:%s",
+ scenario.getName(), scenario.getTableName()));
+ }
+
+ @Override public List<ResultValue<OperationStats>> getResults() {
+ return new ArrayList<>();
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/TenantOperationWorkHandler.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/TenantOperationWorkHandler.java
new file mode 100644
index 0000000..f9195c5
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/handlers/TenantOperationWorkHandler.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.handlers;
+
+import org.apache.phoenix.pherf.workload.mt.MultiTenantWorkload;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.lmax.disruptor.LifecycleAware;
+import com.lmax.disruptor.WorkHandler;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.result.ResultValue;
+import org.apache.phoenix.pherf.workload.mt.operations.OperationStats;
+import org.apache.phoenix.pherf.workload.mt.generators.BaseLoadEventGenerator.TenantOperationEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A handler {@link WorkHandler} for
+ * executing the operations {@link Operation}
+ * as and when they become available on the {@link com.lmax.disruptor.RingBuffer}
+ * when published by the workload generator {@link MultiTenantWorkload}
+ */
+
+public class TenantOperationWorkHandler implements PherfWorkHandler<TenantOperationEvent>,
+ LifecycleAware {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TenantOperationWorkHandler.class);
+ private final String handlerId;
+ private final TenantOperationFactory operationFactory;
+
+ public TenantOperationWorkHandler(TenantOperationFactory operationFactory,
+ String handlerId) {
+ this.handlerId = handlerId;
+ this.operationFactory = operationFactory;
+ }
+
+ @Override
+ public void onEvent(TenantOperationEvent event)
+ throws Exception {
+ TenantOperationInfo input = event.getTenantOperationInfo();
+ Supplier<Function<TenantOperationInfo, OperationStats>> opSupplier =
+ operationFactory.getOperationSupplier(input);
+ OperationStats stats = opSupplier.get().apply(input);
+ stats.setHandlerId(handlerId);
+ // TODO need to handle asynchronous result publishing
+ LOGGER.info(operationFactory.getPhoenixUtil().getGSON().toJson(stats));
+ }
+
+ @Override
+ public void onStart() {
+ Scenario scenario = operationFactory.getScenario();
+ LOGGER.info(String.format("TenantOperationWorkHandler started for %s:%s",
+ scenario.getName(), scenario.getTableName()));
+ }
+
+ @Override
+ public void onShutdown() {
+ Scenario scenario = operationFactory.getScenario();
+ LOGGER.info(String.format("TenantOperationWorkHandler stopped for %s:%s",
+ scenario.getName(), scenario.getTableName()));
+ }
+
+ @Override public List<ResultValue<OperationStats>> getResults() {
+ return new ArrayList<>();
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/BaseOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/BaseOperationSupplier.java
new file mode 100644
index 0000000..034fa61
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/BaseOperationSupplier.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.rules.RulesApplier;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+
+/**
+ * An abstract base class for all OperationSuppliers
+ */
+public abstract class BaseOperationSupplier implements Supplier<Function<TenantOperationInfo, OperationStats>> {
+
+ final PhoenixUtil phoenixUtil;
+ final DataModel model;
+ final Scenario scenario;
+ final RulesApplier rulesApplier;
+ final LoadProfile loadProfile;
+
+ public BaseOperationSupplier(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ this.phoenixUtil = phoenixUtil;
+ this.model = model;
+ this.scenario = scenario;
+ this.rulesApplier = new RulesApplier(model);
+ this.loadProfile = this.scenario.getLoadProfile();
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/IdleTimeOperation.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/IdleTimeOperation.java
new file mode 100644
index 0000000..6dd9262
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/IdleTimeOperation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.IdleTime;
+
+/**
+ * Defines a no op operation, typically used to simulate idle time.
+ * @see {@link OperationType#IDLE_TIME}
+ */
+public interface IdleTimeOperation extends Operation {
+ IdleTime getIdleTime();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/IdleTimeOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/IdleTimeOperationSupplier.java
new file mode 100644
index 0000000..61349c9
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/IdleTimeOperationSupplier.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import com.google.common.base.Function;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.IdleTime;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A supplier of {@link Function} that takes {@link IdleTimeOperation} as an input.
+ */
+public class IdleTimeOperationSupplier extends BaseOperationSupplier {
+ private static final Logger LOGGER = LoggerFactory.getLogger(IdleTimeOperationSupplier.class);
+
+ public IdleTimeOperationSupplier(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ super(phoenixUtil, model, scenario);
+ }
+
+ @Override
+ public Function<TenantOperationInfo, OperationStats> get() {
+
+ return new Function<TenantOperationInfo, OperationStats>() {
+
+ @Override
+ public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
+ final IdleTimeOperation operation = (IdleTimeOperation) input.getOperation();
+ final IdleTime idleTime = operation.getIdleTime();
+
+ final String tenantId = input.getTenantId();
+ final String tenantGroup = input.getTenantGroupId();
+ final String opGroup = input.getOperationGroupId();
+ final String tableName = input.getTableName();
+ final String scenarioName = input.getScenarioName();
+ final String opName = String.format("%s:%s:%s:%s:%s", scenarioName, tableName,
+ opGroup, tenantGroup, tenantId);
+
+ long startTime = EnvironmentEdgeManager.currentTimeMillis();
+ int status = 0;
+
+ // Sleep for the specified time to simulate idle time.
+ try {
+ TimeUnit.MILLISECONDS.sleep(idleTime.getIdleTime());
+ } catch (InterruptedException ie) {
+ LOGGER.error("Operation " + opName + " failed with exception ", ie);
+ status = -1;
+ }
+ long duration = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ return new OperationStats(input, startTime, status, 0, duration);
+ }
+ };
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/Operation.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/Operation.java
new file mode 100644
index 0000000..6556784
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/Operation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+/**
+ * An interface that defines the type of operation included in the load profile.
+ * @see {@link org.apache.phoenix.pherf.configuration.LoadProfile}
+ */
+public interface Operation {
+ enum OperationType {
+ PRE_RUN, UPSERT, SELECT, IDLE_TIME, USER_DEFINED
+ }
+ String getId();
+ OperationType getType();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/OperationStats.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/OperationStats.java
new file mode 100644
index 0000000..8e4a44d
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/OperationStats.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.result.ResultValue;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.operations.Operation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Holds metrics + contextual info on the operation run.
+ */
+public class OperationStats {
+ private final TenantOperationInfo input;
+ private String handlerId;
+ private final int status;
+ private final long rowCount;
+ private final long durationInMs;
+ private final long startTime;
+
+ public OperationStats(
+ TenantOperationInfo input,
+ long startTime,
+ int status,
+ long rowCount,
+ long durationInMs) {
+ this.input = input;
+ this.startTime = startTime;
+ this.status = status;
+ this.rowCount = rowCount;
+ this.durationInMs = durationInMs;
+ }
+
+ public String getModelName() { return this.input.getModelName(); }
+
+ public String getScenarioName() { return this.input.getScenarioName(); }
+
+ public String getTenantId() { return this.input.getTenantId(); }
+
+ public Operation.OperationType getOpType() { return this.input.getOperation().getType(); }
+
+ public String getTableName() {
+ return this.input.getTableName();
+ }
+
+ public String getTenantGroup() {
+ return this.input.getTenantGroupId();
+ }
+
+ public String getOperationGroup() {
+ return this.input.getOperationGroupId();
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public long getRowCount() {
+ return rowCount;
+ }
+
+ public String getHandlerId() { return handlerId; }
+
+ public long getStartTime() { return startTime; }
+
+ public long getDurationInMs() {
+ return durationInMs;
+ }
+
+ public List<ResultValue> getCsvRepresentation() {
+ List<ResultValue> rowValues = new ArrayList<>();
+ rowValues.add(new ResultValue(getModelName()));
+ rowValues.add(new ResultValue(getScenarioName()));
+ rowValues.add(new ResultValue(getTableName()));
+ rowValues.add(new ResultValue(getTenantId()));
+ rowValues.add(new ResultValue(handlerId));
+ rowValues.add(new ResultValue(getTenantGroup()));
+ rowValues.add(new ResultValue(getOperationGroup()));
+ rowValues.add(new ResultValue(getOpType().name()));
+ rowValues.add(new ResultValue(String.valueOf(startTime)));
+ rowValues.add(new ResultValue(String.valueOf(status)));
+ rowValues.add(new ResultValue(String.valueOf(rowCount)));
+ rowValues.add(new ResultValue(String.valueOf(durationInMs)));
+ return rowValues;
+ }
+
+ public void setHandlerId(String handlerId) {
+ this.handlerId = handlerId;
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/PreScenarioOperation.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/PreScenarioOperation.java
new file mode 100644
index 0000000..6494859
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/PreScenarioOperation.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.Ddl;
+
+import java.util.List;
+
+/**
+ * Defines a pre scenario operation.
+ * @see {@link OperationType#PRE_RUN}
+ */
+public interface PreScenarioOperation extends Operation {
+ List<Ddl> getPreScenarioDdls();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/PreScenarioOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/PreScenarioOperationSupplier.java
new file mode 100644
index 0000000..ad423ca
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/PreScenarioOperationSupplier.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import com.google.common.base.Function;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Ddl;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+
+/**
+ * A supplier of {@link Function} that takes {@link PreScenarioOperation} as an input
+ */
+public class PreScenarioOperationSupplier extends BaseOperationSupplier {
+ private static final Logger LOGGER = LoggerFactory.getLogger(PreScenarioOperationSupplier.class);
+
+ public PreScenarioOperationSupplier(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ super(phoenixUtil, model, scenario);
+ }
+
+ @Override
+ public Function<TenantOperationInfo, OperationStats> get() {
+ return new Function<TenantOperationInfo, OperationStats>() {
+
+ @Override
+ public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
+ final PreScenarioOperation operation = (PreScenarioOperation) input.getOperation();
+ final String tenantGroup = input.getTenantGroupId();
+ final String opGroup = input.getOperationGroupId();
+ final String tableName = input.getTableName();
+ final String scenarioName = input.getScenarioName();
+ final boolean isTenantGroupGlobal = (tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0);
+
+ long startTime = EnvironmentEdgeManager.currentTimeMillis();
+ int status = 0;
+ if (!operation.getPreScenarioDdls().isEmpty()) {
+ for (Ddl ddl : operation.getPreScenarioDdls()) {
+ // TODO:
+ // Ideally the fact that the op needs to executed using global connection
+ // needs to be built into the framework and injected during event generation.
+ // For now a special tenant whose id = "TGLOBAL00000001" will be logged.
+ final String tenantId = isTenantGroupGlobal || ddl.isUseGlobalConnection() ? null : input.getTenantId();
+ final String opName = String.format("%s:%s:%s:%s:%s",
+ scenarioName, tableName, opGroup, tenantGroup, input.getTenantId());
+
+ try (Connection conn = phoenixUtil.getConnection(tenantId)) {
+ LOGGER.info("\nExecuting DDL:" + ddl + ", OPERATION:" + opName);
+ String sql = ddl.toString();
+ phoenixUtil.executeStatement(sql, conn);
+ if (ddl.getStatement().toUpperCase().contains(phoenixUtil.ASYNC_KEYWORD)) {
+ phoenixUtil.waitForAsyncIndexToFinish(ddl.getTableName());
+ }
+ } catch (Exception e) {
+ LOGGER.error("Operation " + opName + " failed with exception ", e);
+ status = -1;
+ }
+ }
+ }
+ long totalDuration = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ return new OperationStats(input, startTime, status, operation.getPreScenarioDdls().size(), totalDuration);
+ }
+ };
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/QueryOperation.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/QueryOperation.java
new file mode 100644
index 0000000..db32567
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/QueryOperation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.Query;
+
+/**
+ * Defines a query operation.
+ * @see {@link OperationType#SELECT}
+ */
+public interface QueryOperation extends Operation {
+ Query getQuery();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/QueryOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/QueryOperationSupplier.java
new file mode 100644
index 0000000..0df7836
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/QueryOperationSupplier.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import com.google.common.base.Function;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Query;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+
+/**
+ * A supplier of {@link Function} that takes {@link QueryOperation} as an input.
+ */
+public class QueryOperationSupplier extends BaseOperationSupplier {
+ private static final Logger LOGGER = LoggerFactory.getLogger(QueryOperationSupplier.class);
+
+ public QueryOperationSupplier(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ super(phoenixUtil, model, scenario);
+ }
+
+ @Override
+ public Function<TenantOperationInfo, OperationStats> get() {
+ return new Function<TenantOperationInfo, OperationStats>() {
+
+ @Override
+ public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
+ final QueryOperation operation = (QueryOperation) input.getOperation();
+ final Query query = operation.getQuery();
+ final String tenantGroup = input.getTenantGroupId();
+ final String opGroup = input.getOperationGroupId();
+ final String scenarioName = input.getScenarioName();
+ final String tableName = input.getTableName();
+
+ // TODO:
+ // Ideally the fact that the op needs to executed using global connection
+ // needs to be built into the framework and injected during event generation.
+ // For now a special tenant whose id = "TGLOBAL00000001" will be logged.
+ final boolean isTenantGroupGlobal = (tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0);
+ final String tenantId = isTenantGroupGlobal || query.isUseGlobalConnection() ? null : input.getTenantId();
+
+ String opName = String.format("%s:%s:%s:%s:%s", scenarioName, tableName,
+ opGroup, tenantGroup, input.getTenantId());
+ LOGGER.debug("\nExecuting query " + query.getStatement());
+
+ long startTime = 0;
+ int status = 0;
+ Long resultRowCount = 0L;
+ Long queryElapsedTime = 0L;
+ try (Connection connection = phoenixUtil.getConnection(tenantId)) {
+ startTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ // TODO handle dynamic statements
+ try (PreparedStatement statement = connection.prepareStatement(query.getStatement())) {
+ try (ResultSet rs = statement.executeQuery()) {
+ boolean isSelectCountStatement = query.getStatement().toUpperCase().trim().contains("COUNT(") ? true : false;
+ Pair<Long, Long> r = phoenixUtil.getResults(query, rs, opName,
+ isSelectCountStatement, startTime);
+ resultRowCount = r.getFirst();
+ queryElapsedTime = r.getSecond();
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.error("Operation " + opName + " failed with exception ", e);
+ status = -1;
+ }
+ return new OperationStats(input, startTime, status, resultRowCount, queryElapsedTime);
+ }
+ };
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/TenantOperationFactory.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/TenantOperationFactory.java
new file mode 100644
index 0000000..8559088
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/TenantOperationFactory.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.pherf.workload.mt.generators.LoadEventGenerator;
+import org.apache.phoenix.pherf.workload.mt.MultiTenantWorkload;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.handlers.TenantOperationWorkHandler;
+import com.google.common.base.Charsets;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.hash.BloomFilter;
+import com.google.common.hash.Funnel;
+import com.google.common.hash.PrimitiveSink;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Ddl;
+import org.apache.phoenix.pherf.configuration.IdleTime;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Query;
+import org.apache.phoenix.pherf.configuration.QuerySet;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.configuration.Upsert;
+import org.apache.phoenix.pherf.configuration.UserDefined;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.rules.RulesApplier;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Factory class for operation suppliers.
+ * The class is responsible for creating new instances of suppliers {@link Supplier}
+ * for operations {@link Operation}
+ *
+ * Operations that need to be executed for a given {@link Scenario} and {@link DataModel}
+ * are generated by {@link LoadEventGenerator}
+ *
+ * These operation events are then published on to the {@link com.lmax.disruptor.RingBuffer}
+ * by the {@link MultiTenantWorkload} workload generator and
+ * handled by the {@link com.lmax.disruptor.WorkHandler} for eg {@link TenantOperationWorkHandler}
+ */
+public class TenantOperationFactory {
+
+ private static class TenantView {
+ private final String tenantId;
+ private final String viewName;
+
+ public TenantView(String tenantId, String viewName) {
+ this.tenantId = tenantId;
+ this.viewName = viewName;
+ }
+
+ public String getTenantId() {
+ return tenantId;
+ }
+
+ public String getViewName() {
+ return viewName;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ TenantView that = (TenantView) o;
+ return getTenantId().equals(that.getTenantId()) && getViewName()
+ .equals(that.getViewName());
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(getTenantId(), getViewName());
+ }
+ }
+ private static final Logger LOGGER = LoggerFactory.getLogger(TenantOperationFactory.class);
+ private final PhoenixUtil phoenixUtil;
+ private final DataModel model;
+ private final Scenario scenario;
+ private final XMLConfigParser parser;
+
+ private final RulesApplier rulesApplier;
+ private final LoadProfile loadProfile;
+ private final List<Operation> operationList = Lists.newArrayList();
+ private final Map<Operation.OperationType, Supplier<Function<TenantOperationInfo, OperationStats>>> operationSuppliers =
+ Maps.newEnumMap(Operation.OperationType.class);
+
+ private final BloomFilter<TenantView> tenantsLoaded;
+ private final ReentrantReadWriteLock viewCreationLock = new ReentrantReadWriteLock();
+
+ public TenantOperationFactory(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ this.phoenixUtil = phoenixUtil;
+ this.model = model;
+ this.scenario = scenario;
+ this.parser = null;
+ this.rulesApplier = new RulesApplier(model);
+ this.loadProfile = this.scenario.getLoadProfile();
+ this.tenantsLoaded = createTenantsLoadedFilter(loadProfile);
+
+ // Read the scenario definition and load the various operations.
+ // Case : Operation.OperationType.PRE_RUN
+ if (scenario.getPreScenarioDdls() != null && scenario.getPreScenarioDdls().size() > 0) {
+ operationSuppliers.put(Operation.OperationType.PRE_RUN,
+ new PreScenarioOperationSupplier(phoenixUtil, model, scenario));
+ }
+
+ // Case : Operation.OperationType.UPSERT
+ List<Operation> upsertOperations = getUpsertOperationsForScenario(scenario);
+ if (upsertOperations.size() > 0) {
+ operationList.addAll(upsertOperations);
+ operationSuppliers.put(Operation.OperationType.UPSERT,
+ new UpsertOperationSupplier(phoenixUtil, model, scenario));
+ }
+
+ // Case : Operation.OperationType.SELECT
+ List<Operation> queryOperations = getQueryOperationsForScenario(scenario);
+ if (queryOperations.size() > 0) {
+ operationList.addAll(queryOperations);
+ operationSuppliers.put(Operation.OperationType.SELECT,
+ new QueryOperationSupplier(phoenixUtil, model, scenario));
+ }
+
+ // Case : Operation.OperationType.IDLE_TIME
+ List<Operation> idleOperations = getIdleTimeOperationsForScenario(scenario);
+ if (idleOperations.size() > 0) {
+ operationList.addAll(idleOperations);
+ operationSuppliers.put(Operation.OperationType.IDLE_TIME,
+ new IdleTimeOperationSupplier(phoenixUtil, model, scenario));
+ }
+
+ // Case : Operation.OperationType.USER_DEFINED
+ List<Operation> udfOperations = getUDFOperationsForScenario(scenario);
+ if (udfOperations.size() > 0) {
+ operationList.addAll(udfOperations);
+ operationSuppliers.put(Operation.OperationType.USER_DEFINED,
+ new UserDefinedOperationSupplier(phoenixUtil, model, scenario));
+ }
+ }
+
+ private BloomFilter createTenantsLoadedFilter(LoadProfile loadProfile) {
+ Funnel<TenantView> tenantViewFunnel = new Funnel<TenantView>() {
+ @Override
+ public void funnel(TenantView tenantView, PrimitiveSink into) {
+ into.putString(tenantView.getTenantId(), Charsets.UTF_8)
+ .putString(tenantView.getViewName(), Charsets.UTF_8);
+ }
+ };
+
+ int numTenants = 0;
+ for (TenantGroup tg : loadProfile.getTenantDistribution()) {
+ numTenants += tg.getNumTenants();
+ }
+
+ // This holds the info whether the tenant view was created (initialized) or not.
+ return BloomFilter.create(tenantViewFunnel, numTenants, 0.0000001);
+ }
+
+ private List<Operation> getUpsertOperationsForScenario(Scenario scenario) {
+ List<Operation> opList = Lists.newArrayList();
+ for (final Upsert upsert : scenario.getUpserts()) {
+ final Operation upsertOp = new UpsertOperation() {
+ @Override public Upsert getUpsert() {
+ return upsert;
+ }
+
+ @Override public String getId() {
+ return upsert.getId();
+ }
+
+ @Override public OperationType getType() {
+ return OperationType.UPSERT;
+ }
+ };
+ opList.add(upsertOp);
+ }
+ return opList;
+ }
+
+ private List<Operation> getQueryOperationsForScenario(Scenario scenario) {
+ List<Operation> opList = Lists.newArrayList();
+ for (final QuerySet querySet : scenario.getQuerySet()) {
+ for (final Query query : querySet.getQuery()) {
+ Operation queryOp = new QueryOperation() {
+ @Override public Query getQuery() {
+ return query;
+ }
+
+ @Override public String getId() {
+ return query.getId();
+ }
+
+ @Override public OperationType getType() {
+ return OperationType.SELECT;
+ }
+ };
+ opList.add(queryOp);
+ }
+ }
+ return opList;
+ }
+
+ private List<Operation> getIdleTimeOperationsForScenario(Scenario scenario) {
+ List<Operation> opList = Lists.newArrayList();
+ for (final IdleTime idleTime : scenario.getIdleTimes()) {
+ Operation idleTimeOperation = new IdleTimeOperation() {
+ @Override public IdleTime getIdleTime() {
+ return idleTime;
+ }
+ @Override public String getId() {
+ return idleTime.getId();
+ }
+
+ @Override public OperationType getType() {
+ return OperationType.IDLE_TIME;
+ }
+ };
+ opList.add(idleTimeOperation);
+ }
+ return opList;
+ }
+
+ private List<Operation> getUDFOperationsForScenario(Scenario scenario) {
+ List<Operation> opList = Lists.newArrayList();
+ for (final UserDefined udf : scenario.getUdfs()) {
+ Operation udfOperation = new UserDefinedOperation() {
+ @Override public UserDefined getUserFunction() {
+ return udf;
+ }
+
+ @Override public String getId() {
+ return udf.getId();
+ }
+
+ @Override public OperationType getType() {
+ return OperationType.USER_DEFINED;
+ }
+ };
+ opList.add(udfOperation);
+ }
+ return opList;
+ }
+
+ public PhoenixUtil getPhoenixUtil() {
+ return phoenixUtil;
+ }
+
+ public DataModel getModel() {
+ return model;
+ }
+
+ public Scenario getScenario() {
+ return scenario;
+ }
+
+ public List<Operation> getOperations() {
+ return operationList;
+ }
+
+ public void initializeTenant(TenantOperationInfo input) throws Exception {
+ TenantView tenantView = new TenantView(input.getTenantId(), scenario.getTableName());
+
+ // Check if pre run ddls are needed.
+ viewCreationLock.writeLock().lock();
+ try {
+ if (!tenantsLoaded.mightContain(tenantView)) {
+ executePreRunOpsForTenant(tenantView, input);
+ boolean updated = tenantsLoaded.put(tenantView);
+ if (updated) {
+ LOGGER.info(String.format("Successfully initialized tenant. [%s, %s] ",
+ tenantView.tenantId,
+ tenantView.viewName));
+ }
+ }
+ } finally {
+ viewCreationLock.writeLock().unlock();
+ }
+ }
+
+
+ public Supplier<Function<TenantOperationInfo, OperationStats>> getOperationSupplier(
+ final TenantOperationInfo input) throws Exception {
+
+ Supplier<Function<TenantOperationInfo, OperationStats>> opSupplier =
+ operationSuppliers.get(input.getOperation().getType());
+ if (opSupplier == null) {
+ throw new IllegalArgumentException("Unknown operation type");
+ }
+ return opSupplier;
+ }
+
+ private void executePreRunOpsForTenant(TenantView tenantView, TenantOperationInfo input) throws Exception {
+
+ Supplier<Function<TenantOperationInfo, OperationStats>> preRunOpSupplier =
+ operationSuppliers.get(Operation.OperationType.PRE_RUN);
+ // Check if the scenario has a PRE_RUN operation.
+ if (preRunOpSupplier != null) {
+ // Initialize the tenant using the pre scenario ddls.
+ final PreScenarioOperation
+ operation = new PreScenarioOperation() {
+ @Override public List<Ddl> getPreScenarioDdls() {
+ List<Ddl> ddls = scenario.getPreScenarioDdls();
+ return ddls == null ? Lists.<Ddl>newArrayList() : ddls;
+ }
+
+ @Override public String getId() {
+ return OperationType.PRE_RUN.name();
+ }
+
+ @Override public OperationType getType() {
+ return OperationType.PRE_RUN;
+ }
+ };
+ // Initialize with the pre run operation.
+ TenantOperationInfo preRunSample = new TenantOperationInfo(
+ input.getModelName(),
+ input.getScenarioName(),
+ input.getTableName(),
+ input.getTenantGroupId(),
+ Operation.OperationType.PRE_RUN.name(),
+ input.getTenantId(), operation);
+
+ try {
+ // Run the initialization operation.
+ OperationStats stats = preRunOpSupplier.get().apply(preRunSample);
+ LOGGER.info(phoenixUtil.getGSON().toJson(stats));
+ } catch (Exception e) {
+ LOGGER.error(String.format("Failed to initialize tenant. [%s, %s] ",
+ tenantView.tenantId,
+ tenantView.viewName), e);
+ if (e.getClass().isAssignableFrom(SQLException.class)) {
+ SQLException sqlException = (SQLException) e;
+ if (SQLExceptionCode.CONCURRENT_TABLE_MUTATION.getErrorCode() != sqlException.getErrorCode()) {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ }
+ }
+ }
+
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UpsertOperation.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UpsertOperation.java
new file mode 100644
index 0000000..9c7d59b
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UpsertOperation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.Upsert;
+
+/**
+ * Defines an upsert operation.
+ * @see {@link OperationType#UPSERT}
+ */
+public interface UpsertOperation extends Operation {
+ Upsert getUpsert();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UpsertOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UpsertOperationSupplier.java
new file mode 100644
index 0000000..c8e9f3d
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UpsertOperationSupplier.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.TenantGroup;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import com.google.common.base.Function;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.Upsert;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * A supplier of {@link Function} that takes {@link UpsertOperation} as an input
+ */
+public class UpsertOperationSupplier extends BaseOperationSupplier {
+ private static final Logger LOGGER = LoggerFactory.getLogger(UpsertOperationSupplier.class);
+ private ReadWriteLock rwLock = new ReentrantReadWriteLock();
+
+ public UpsertOperationSupplier(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ super(phoenixUtil, model, scenario);
+ }
+ @Override
+ public Function<TenantOperationInfo, OperationStats> get() {
+ return new Function<TenantOperationInfo, OperationStats>() {
+
+ @Override
+ public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
+ final int batchSize = loadProfile.getBatchSize();
+ final boolean useBatchApi = batchSize != 0;
+ final int rowCount = useBatchApi ? batchSize : 1;
+
+ final UpsertOperation operation = (UpsertOperation) input.getOperation();
+ final Upsert upsert = operation.getUpsert();
+ final String tenantGroup = input.getTenantGroupId();
+ final String opGroup = input.getOperationGroupId();
+ final String tableName = input.getTableName();
+ final String scenarioName = input.getScenarioName();
+
+ // TODO:
+ // Ideally the fact that the op needs to executed using global connection
+ // needs to be built into the framework and injected during event generation.
+ // For now a special tenant whose id = "TGLOBAL00000001" will be logged.
+
+ final boolean isTenantGroupGlobal = (tenantGroup.compareTo(TenantGroup.DEFAULT_GLOBAL_ID) == 0);
+ final String tenantId = isTenantGroupGlobal || upsert.isUseGlobalConnection() ? null : input.getTenantId();
+ final String opName = String.format("%s:%s:%s:%s:%s",
+ scenarioName, tableName, opGroup, tenantGroup, input.getTenantId());
+ long rowsCreated = 0;
+ long startTime = 0, duration, totalDuration;
+ int status = 0;
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+ try (Connection connection = phoenixUtil.getConnection(tenantId)) {
+ // If list of columns has not been not provided or lazy loaded
+ // then use the metadata call to get the column list.
+ if (upsert.getColumn().isEmpty()) {
+ rwLock.writeLock().lock();
+ try {
+ if (upsert.getColumn().isEmpty()) {
+ LOGGER.info("Fetching columns metadata from db for operation : " + opName);
+ List<Column> allCols = phoenixUtil.getColumnsFromPhoenix(scenario.getSchemaName(),
+ scenario.getTableNameWithoutSchemaName(),
+ connection);
+ upsert.setColumn(allCols);
+ }
+ } finally {
+ rwLock.writeLock().unlock();
+ }
+ }
+
+ String sql = phoenixUtil.buildSql(upsert.getColumn(), tableName);
+ LOGGER.info("Operation " + opName + " executing " + sql);
+ startTime = EnvironmentEdgeManager.currentTimeMillis();
+ PreparedStatement stmt = null;
+ try {
+ stmt = connection.prepareStatement(sql);
+ for (long i = rowCount; i > 0; i--) {
+ stmt = phoenixUtil.buildStatement(rulesApplier, scenario, upsert.getColumn(), stmt, simpleDateFormat);
+ if (useBatchApi) {
+ stmt.addBatch();
+ } else {
+ rowsCreated += stmt.executeUpdate();
+ }
+ }
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ // Need to keep the statement open to send the remaining batch of updates
+ if (!useBatchApi && stmt != null) {
+ stmt.close();
+ }
+ if (connection != null) {
+ if (useBatchApi && stmt != null) {
+ int[] results = stmt.executeBatch();
+ for (int x = 0; x < results.length; x++) {
+ int result = results[x];
+ if (result < 1) {
+ final String msg =
+ "Failed to write update in batch (update count="
+ + result + ")";
+ throw new RuntimeException(msg);
+ }
+ rowsCreated += result;
+ }
+ // Close the statement after our last batch execution.
+ stmt.close();
+ }
+
+ try {
+ connection.commit();
+ duration = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ LOGGER.info("Writer ( " + Thread.currentThread().getName()
+ + ") committed Final Batch. Duration (" + duration + ") Ms");
+ connection.close();
+ } catch (SQLException e) {
+ // Swallow since we are closing anyway
+ LOGGER.error("Error when closing/committing", e);
+ }
+ }
+ }
+ } catch (SQLException sqle) {
+ LOGGER.error("Operation " + opName + " failed with exception ", sqle);
+ status = -1;
+ } catch (Exception e) {
+ LOGGER.error("Operation " + opName + " failed with exception ", e);
+ status = -1;
+ }
+
+ totalDuration = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ return new OperationStats(input, startTime, status, rowsCreated, totalDuration);
+ }
+ };
+ }
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UserDefinedOperation.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UserDefinedOperation.java
new file mode 100644
index 0000000..838666c
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UserDefinedOperation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.configuration.UserDefined;
+
+/**
+ * Defines an user defined operation.
+ * @see {@link OperationType#USER_DEFINED}
+ */
+public interface UserDefinedOperation extends Operation {
+ UserDefined getUserFunction();
+}
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UserDefinedOperationSupplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UserDefinedOperationSupplier.java
new file mode 100644
index 0000000..8037bd3
--- /dev/null
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/mt/operations/UserDefinedOperationSupplier.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt.operations;
+
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import com.google.common.base.Function;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import com.google.common.base.Preconditions;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+
+/**
+ * A supplier of {@link Function} that takes {@link UserDefinedOperation} as an input
+ */
+public class UserDefinedOperationSupplier extends BaseOperationSupplier {
+
+ public UserDefinedOperationSupplier(PhoenixUtil phoenixUtil, DataModel model, Scenario scenario) {
+ super(phoenixUtil, model, scenario);
+ }
+
+ @Override
+ public Function<TenantOperationInfo, OperationStats> get() {
+ return new Function<TenantOperationInfo, OperationStats>() {
+ @Override
+ public OperationStats apply(final TenantOperationInfo input) {
+ Preconditions.checkNotNull(input);
+ // TODO : implement user defined operation invocation.
+ long startTime = EnvironmentEdgeManager.currentTimeMillis();
+ long duration = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ return new OperationStats(input, startTime,0, 0, duration);
+ }
+ };
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
index 343285f..e3133f7 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ConfigurationParserTest.java
@@ -24,9 +24,25 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
-import org.apache.phoenix.pherf.configuration.*;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataOverride;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+import org.apache.phoenix.pherf.configuration.Ddl;
+import org.apache.phoenix.pherf.configuration.ExecutionType;
+import org.apache.phoenix.pherf.configuration.Query;
+import org.apache.phoenix.pherf.configuration.QuerySet;
+import org.apache.phoenix.pherf.configuration.WriteParams;
+import com.google.common.collect.Sets;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
import org.apache.phoenix.pherf.rules.DataValue;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantLoadEventGeneratorFactory;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +59,8 @@ public class ConfigurationParserTest extends ResultBaseTest {
@Test
public void testReadWriteWorkloadReader() throws Exception {
String scenarioName = "testScenarioRW";
- List<Scenario> scenarioList = getScenarios();
+ String testResourceName = "/scenario/test_scenario.xml";
+ List<Scenario> scenarioList = getScenarios(testResourceName);
Scenario target = null;
for (Scenario scenario : scenarioList) {
if (scenarioName.equals(scenario.getName())) {
@@ -64,10 +81,10 @@ public class ConfigurationParserTest extends ResultBaseTest {
// TODO Break this into multiple smaller tests.
public void testConfigReader() {
try {
-
+ String testResourceName = "/scenario/test_scenario.xml";
LOGGER.debug("DataModel: " + writeXML());
- List<Scenario> scenarioList = getScenarios();
- List<Column> dataMappingColumns = getDataModel().getDataMappingColumns();
+ List<Scenario> scenarioList = getScenarios(testResourceName);
+ List<Column> dataMappingColumns = getDataModel(testResourceName).getDataMappingColumns();
assertTrue("Could not load the data columns from xml.",
(dataMappingColumns != null) && (dataMappingColumns.size() > 0));
assertTrue("Could not load the data DataValue list from xml.",
@@ -122,29 +139,101 @@ public class ConfigurationParserTest extends ResultBaseTest {
}
}
- private URL getResourceUrl() {
- URL resourceUrl = getClass().getResource("/scenario/test_scenario.xml");
+ @Test
+ public void testWorkloadWithLoadProfile() throws Exception {
+ String testResourceName = "/scenario/test_workload_with_load_profile.xml";
+ Set<String> scenarioNames = Sets.newHashSet("scenario_11", "scenario_12");
+ List<Scenario> scenarioList = getScenarios(testResourceName);
+ Scenario target = null;
+ for (Scenario scenario : scenarioList) {
+ if (scenarioNames.contains(scenario.getName())) {
+ target = scenario;
+ }
+ assertNotNull("Could not find scenario: " + scenario.getName(), target);
+ }
+
+ Scenario testScenarioWithLoadProfile = scenarioList.get(0);
+ Map<String, String> props = testScenarioWithLoadProfile.getPhoenixProperties();
+ assertEquals("Number of properties(size) not as expected: ",
+ 2, props.size());
+ TenantLoadEventGeneratorFactory.GeneratorType
+ type = TenantLoadEventGeneratorFactory.GeneratorType.valueOf(
+ testScenarioWithLoadProfile.getGeneratorName());
+ assertEquals("Unknown generator type: ",
+ TenantLoadEventGeneratorFactory.GeneratorType.UNIFORM, type);
+
+ LoadProfile loadProfile = testScenarioWithLoadProfile.getLoadProfile();
+ assertEquals("batch size not as expected: ",
+ 1, loadProfile.getBatchSize());
+ assertEquals("num operations not as expected: ",
+ 1000, loadProfile.getNumOperations());
+ assertEquals("tenant group size is not as expected: ",
+ 3, loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ",
+ 5,loadProfile.getOpDistribution().size());
+ assertEquals("UDFs size is not as expected ",
+ 1, testScenarioWithLoadProfile.getUdfs().size());
+ assertNotNull("UDFs clazzName cannot be null ",
+ testScenarioWithLoadProfile.getUdfs().get(0).getClazzName());
+ assertEquals("UDFs args size is not as expected ",
+ 2, testScenarioWithLoadProfile.getUdfs().get(0).getArgs().size());
+ assertEquals("UpsertSet size is not as expected ",
+ 1, testScenarioWithLoadProfile.getUpserts().size());
+ assertEquals("#Column within the first upsert is not as expected ",
+ 7, testScenarioWithLoadProfile.getUpserts().get(0).getColumn().size());
+ assertEquals("QuerySet size is not as expected ",
+ 1, testScenarioWithLoadProfile.getQuerySet().size());
+ assertEquals("#Queries within the first querySet is not as expected ",
+ 2, testScenarioWithLoadProfile.getQuerySet().get(0).getQuery().size());
+
+ // Test configuration for global connection
+ Scenario testScenarioWithGlobalConn = scenarioList.get(2);
+ LoadProfile loadProfileWithGlobalConn = testScenarioWithGlobalConn.getLoadProfile();
+ assertEquals("batch size not as expected: ",
+ 1, loadProfileWithGlobalConn.getBatchSize());
+ assertEquals("num operations not as expected: ",
+ 1000, loadProfileWithGlobalConn.getNumOperations());
+ assertEquals("tenant group size is not as expected: ",
+ 1, loadProfileWithGlobalConn.getTenantDistribution().size());
+ assertEquals("global tenant is not as expected: ",
+ 1, loadProfileWithGlobalConn.getTenantDistribution().get(0).getNumTenants());
+ assertEquals("global tenant id is not as expected: ",
+ "GLOBAL", loadProfileWithGlobalConn.getTenantDistribution().get(0).getId());
+ assertEquals("global tenant weight is not as expected: ",
+ 100, loadProfileWithGlobalConn.getTenantDistribution().get(0).getWeight());
+ assertEquals("operation group size is not as expected: ",
+ 1,loadProfileWithGlobalConn.getOpDistribution().size());
+ assertEquals("UpsertSet size is not as expected ",
+ 1, testScenarioWithGlobalConn.getUpserts().size());
+ assertEquals("#Column within the first upsert is not as expected ",
+ 7, testScenarioWithGlobalConn.getUpserts().get(0).getColumn().size());
+ assertEquals("Upsert operation not using global connection as expected ",
+ true, testScenarioWithGlobalConn.getUpserts().get(0).isUseGlobalConnection());
+ }
+
+ private URL getResourceUrl(String resourceName) {
+ URL resourceUrl = getClass().getResource(resourceName);
assertNotNull("Test data XML file is missing", resourceUrl);
return resourceUrl;
}
- private List<Scenario> getScenarios() throws Exception {
- DataModel data = getDataModel();
+ private List<Scenario> getScenarios(String resourceName) throws Exception {
+ DataModel data = getDataModel(resourceName);
List<Scenario> scenarioList = data.getScenarios();
assertTrue("Could not load the scenarios from xml.",
(scenarioList != null) && (scenarioList.size() > 0));
return scenarioList;
}
- private DataModel getDataModel() throws Exception {
- Path resourcePath = Paths.get(getResourceUrl().toURI());
+ private DataModel getDataModel(String resourceName) throws Exception {
+ Path resourcePath = Paths.get(getResourceUrl(resourceName).toURI());
return XMLConfigParser.readDataModel(resourcePath);
}
private void assertDateValue(List<Column> dataMappingColumns) {
for (Column dataMapping : dataMappingColumns) {
if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName()
- .equals("CREATED_DATE"))) {
+ .equals("SOME_DATE"))) {
// First rule should have min/max set
assertNotNull(dataMapping.getDataValues().get(0).getMinValue());
assertNotNull(dataMapping.getDataValues().get(0).getMaxValue());
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
index 6a24ced..b10593e 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/PherfTest.java
@@ -38,7 +38,7 @@ public class PherfTest {
@Test
public void testUnknownOption() {
- String[] args = {"-drop", "all", "-q", "-m","-bsOption"};
+ String[] args = {"-drop", "all", "-q", "-m","-unknownOption"};
// Makes sure that System.exit(1) is called.
exit.expectSystemExitWithStatus(1);
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResultBaseTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResultBaseTest.java
index 1853b67..a4b7648 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResultBaseTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/ResultBaseTest.java
@@ -42,6 +42,10 @@ public class ResultBaseTest {
}
@AfterClass public static synchronized void tearDown() throws Exception {
- new ResultUtil().deleteDir(properties.getProperty("pherf.default.results.dir"));
+ try {
+ new ResultUtil().deleteDir(properties.getProperty("pherf.default.results.dir"));
+ } catch (Exception e) {
+ // swallow
+ }
}
}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
index c439d38..c62922b 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
@@ -59,7 +59,7 @@ public class RuleGeneratorTest {
RulesApplier rulesApplier = loader.getRulesApplier();
for (Column dataMapping : model.getDataMappingColumns()) {
- if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName().equals("CREATED_DATE"))) {
+ if ((dataMapping.getType() == DataTypeMapping.DATE) && (dataMapping.getName().equals("SOME_DATE"))) {
// Test directly through generator method and that it converts to Phoenix type
assertRandomDateValue(dataMapping, rulesApplier);
@@ -92,7 +92,8 @@ public class RuleGeneratorTest {
for (Column dataMapping : model.getDataMappingColumns()) {
if ((dataMapping.getType() == DataTypeMapping.DATE)
- && (dataMapping.getUseCurrentDate() == true)) {
+ && (dataMapping.getUseCurrentDate() == true)
+ && (dataMapping.getDataSequence() != DataSequence.SEQUENTIAL)) {
// Generate the date using rules
DataValue value = rulesApplier.getDataValue(dataMapping);
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java
new file mode 100644
index 0000000..3cc75e4
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialDateDataGeneratorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.joda.time.LocalDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.junit.Test;
+
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.DATE;
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class SequentialDateDataGeneratorTest {
+ SequentialDateDataGenerator generator;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonSequential() {
+ Column columnA = new Column();
+ columnA.setType(DATE);
+ columnA.setDataSequence(DataSequence.RANDOM);
+
+ //should reject this Column
+ generator = new SequentialDateDataGenerator(columnA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonDate() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+
+ //should reject this Column
+ generator = new SequentialDateDataGenerator(columnA);
+ }
+
+ @Test
+ public void testGetDataValue() {
+ DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+ Column columnA = new Column();
+ columnA.setType(DATE);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+ LocalDateTime startDateTime = new LocalDateTime();
+
+ // The increments are the of 1 sec units
+ generator = new SequentialDateDataGenerator(columnA);
+ DataValue result1 = generator.getDataValue();
+ LocalDateTime result1LocalTime = FMT.parseDateTime(result1.getValue()).toLocalDateTime();
+ assertFalse(result1LocalTime.isBefore(startDateTime));
+ DataValue result2 = generator.getDataValue();
+ LocalDateTime result2LocalTime = FMT.parseDateTime(result2.getValue()).toLocalDateTime();
+ assertEquals(result2LocalTime.minusSeconds(1), result1LocalTime);
+ DataValue result3 = generator.getDataValue();
+ LocalDateTime result3LocalTime = FMT.parseDateTime(result3.getValue()).toLocalDateTime();
+ assertEquals(result3LocalTime.minusSeconds(1), result2LocalTime);
+ DataValue result4 = generator.getDataValue();
+ LocalDateTime result4LocalTime = FMT.parseDateTime(result4.getValue()).toLocalDateTime();
+ assertEquals(result4LocalTime.minusSeconds(1), result3LocalTime);
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java
new file mode 100644
index 0000000..d590167
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialListDataGeneratorTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.joda.time.LocalDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.DATE;
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR;
+import static org.junit.Assert.assertEquals;
+
+public class SequentialListDataGeneratorTest {
+ SequentialListDataGenerator generator;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonSequential() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.RANDOM);
+
+ //should reject this Column
+ generator = new SequentialListDataGenerator(columnA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonVarchar() {
+ DateTimeFormatter FMT = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
+ LocalDateTime startDateTime = new LocalDateTime();
+ String formattedDateTime = startDateTime.toString(FMT);
+ Column columnA = new Column();
+ columnA.setType(DATE);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+ List<DataValue> values = new ArrayList<>();
+ values.add(new DataValue(DATE, formattedDateTime));
+ values.add(new DataValue(DATE, formattedDateTime));
+ values.add(new DataValue(DATE, formattedDateTime));
+ columnA.setDataValues(values);
+
+ //should reject this Column
+ generator = new SequentialListDataGenerator(columnA);
+ }
+
+ @Test
+ public void testGetDataValue() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+ List<DataValue> values = new ArrayList<>();
+ values.add(new DataValue(VARCHAR, "A"));
+ values.add(new DataValue(VARCHAR, "B"));
+ values.add(new DataValue(VARCHAR, "C"));
+ columnA.setDataValues(values);
+
+ generator = new SequentialListDataGenerator(columnA);
+ DataValue result1 = generator.getDataValue();
+ assertEquals("A", result1.getValue());
+ DataValue result2 = generator.getDataValue();
+ assertEquals("B", result2.getValue());
+ DataValue result3 = generator.getDataValue();
+ assertEquals("C", result3.getValue());
+ DataValue result4 = generator.getDataValue();
+ assertEquals("A", result4.getValue());
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java
new file mode 100644
index 0000000..0157721
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/rules/SequentialVarcharDataGeneratorTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.pherf.rules;
+
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataSequence;
+import org.junit.Test;
+
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.INTEGER;
+import static org.apache.phoenix.pherf.configuration.DataTypeMapping.VARCHAR;
+import static org.junit.Assert.assertEquals;
+
+public class SequentialVarcharDataGeneratorTest {
+ SequentialVarcharDataGenerator generator;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonSequential() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setDataSequence(DataSequence.RANDOM);
+
+ //should reject this Column
+ generator = new SequentialVarcharDataGenerator(columnA);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRejectsNonVarchar() {
+ Column columnA = new Column();
+ columnA.setType(INTEGER);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+
+ //should reject this Column
+ generator = new SequentialVarcharDataGenerator(columnA);
+ }
+
+ @Test
+ public void testGetDataValue() {
+ Column columnA = new Column();
+ columnA.setType(VARCHAR);
+ columnA.setLength(15);
+ columnA.setDataSequence(DataSequence.SEQUENTIAL);
+
+ generator = new SequentialVarcharDataGenerator(columnA);
+ DataValue result1 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx0", result1.getValue());
+ DataValue result2 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx1", result2.getValue());
+ DataValue result3 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx2", result3.getValue());
+ DataValue result4 = generator.getDataValue();
+ assertEquals("xxxxxxxxxxxxxx3", result4.getValue());
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/SequentialLoadEventGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/SequentialLoadEventGeneratorTest.java
new file mode 100644
index 0000000..b8c4368
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/SequentialLoadEventGeneratorTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.XMLConfigParserTest;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.generators.SequentialLoadEventGenerator;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the various sequential event generation outcomes based on scenario, model
+ * execution type and iterations
+ */
+public class SequentialLoadEventGeneratorTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(
+ SequentialLoadEventGeneratorTest.class);
+
+ private enum TestOperationGroup {
+ upsertOp, queryOp1, queryOp2, queryOp3, queryOp4, queryOp5, queryOp6, queryOp7, idleOp, udfOp
+ }
+
+ private enum TestTenantGroup {
+ tg1
+ }
+
+ public DataModel readTestDataModel(String resourceName) throws Exception {
+ URL scenarioUrl = XMLConfigParserTest.class.getResource(resourceName);
+ assertNotNull(scenarioUrl);
+ Path p = Paths.get(scenarioUrl.toURI());
+ return XMLConfigParser.readDataModel(p);
+ }
+
+ @Test
+ public void testParallelExecutionWithOneHandler() throws Exception {
+ sequentialEventGeneration(1, true);
+ }
+
+ @Test
+ public void testParallelExecutionWithManyHandler() throws Exception {
+ sequentialEventGeneration(5, true);
+ }
+
+ @Test
+ public void testSerialExecutionWithOneHandler() throws Exception {
+ sequentialEventGeneration(1, false);
+ }
+
+ @Test
+ public void testSerialExecutionWithManyHandler() throws Exception {
+ sequentialEventGeneration(5, false);
+ }
+
+ public void sequentialEventGeneration(int numIterations, boolean parallel) throws Exception {
+ int numTenantGroups = 1;
+ int numOpGroups = 10;
+ double variancePercent = 0.00f; // 0 percent
+
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ Properties properties = PherfConstants
+ .create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+ properties.setProperty(PherfConstants.NUM_SEQUENTIAL_ITERATIONS_PROP_KEY, String.valueOf(numIterations));
+ properties.setProperty(PherfConstants.NUM_SEQUENTIAL_EXECUTION_TYPE_PROP_KEY,
+ parallel ? "PARALLEL" : "SERIAL");
+
+ DataModel model = readTestDataModel("/scenario/test_evt_gen4.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+ assertEquals("tenant group size is not as expected: ",
+ numTenantGroups, loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ",
+ numOpGroups, loadProfile.getOpDistribution().size());
+ // Calculate the expected distribution.
+ double[][] expectedDistribution = new double[numOpGroups][numTenantGroups];
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ expectedDistribution[r][c] = numIterations;
+ LOGGER.debug(String.format("Expected [%d,%d] = %f", r, c, expectedDistribution[r][c]));
+ }
+ }
+
+ SequentialLoadEventGenerator evtGen = new SequentialLoadEventGenerator(
+ pUtil, model, scenario, properties);
+
+ // Calculate the actual distribution.
+ double[][] distribution = new double[numOpGroups][numTenantGroups];
+ for (int i = 0; i < numIterations; i++) {
+ for (int r = 0; r < numOpGroups; r++) {
+ TenantOperationInfo info = evtGen.next();
+ int row = TestOperationGroup.valueOf(info.getOperationGroupId()).ordinal();
+ int col = TestTenantGroup.valueOf(info.getTenantGroupId()).ordinal();
+ distribution[row][col]++;
+ }
+ }
+
+ // Validate that the expected and actual distribution
+ // is within the margin of allowed variance.
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ double allowedVariance = expectedDistribution[r][c] * variancePercent;
+ double diff = Math.abs(expectedDistribution[r][c] - distribution[r][c]);
+ boolean isAllowed = diff == allowedVariance;
+ LOGGER.debug(String.format("Actual[%d,%d] = %f, %f, %f",
+ r, c, distribution[r][c], diff, allowedVariance));
+ assertTrue(String.format("Difference is outside the allowed variance "
+ + "[expected = %f, actual = %f]", allowedVariance, diff), isAllowed);
+
+ }
+ }
+ }
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/TenantOperationFactoryTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/TenantOperationFactoryTest.java
new file mode 100644
index 0000000..5dd3935
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/TenantOperationFactoryTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.XMLConfigParserTest;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.generators.WeightedRandomLoadEventGenerator;
+import org.apache.phoenix.pherf.workload.mt.operations.IdleTimeOperationSupplier;
+import org.apache.phoenix.pherf.workload.mt.operations.QueryOperationSupplier;
+import org.apache.phoenix.pherf.workload.mt.operations.TenantOperationFactory;
+import org.apache.phoenix.pherf.workload.mt.operations.UpsertOperationSupplier;
+import org.apache.phoenix.pherf.workload.mt.operations.UserDefinedOperationSupplier;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Tests the various operation supplier outcomes based on scenario, model and load profile.
+ */
+public class TenantOperationFactoryTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TenantOperationFactoryTest.class);
+
+ private enum TestOperationGroup {
+ upsertOp, queryOp1, queryOp2, idleOp, udfOp
+ }
+
+ private enum TestTenantGroup {
+ tg1, tg2, tg3
+ }
+
+ public DataModel readTestDataModel(String resourceName) throws Exception {
+ URL scenarioUrl = XMLConfigParserTest.class.getResource(resourceName);
+ assertNotNull(scenarioUrl);
+ Path p = Paths.get(scenarioUrl.toURI());
+ return XMLConfigParser.readDataModel(p);
+ }
+
+ @Test public void testVariousOperations() throws Exception {
+ int numTenantGroups = 3;
+ int numOpGroups = 5;
+ int numRuns = 10;
+ int numOperations = 10;
+
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ Properties properties = PherfConstants
+ .create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+
+ DataModel model = readTestDataModel("/scenario/test_evt_gen1.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+ assertEquals("tenant group size is not as expected: ",
+ numTenantGroups, loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ",
+ numOpGroups, loadProfile.getOpDistribution().size());
+
+ WeightedRandomLoadEventGenerator evtGen = new WeightedRandomLoadEventGenerator(
+ pUtil, model, scenario, properties);
+ TenantOperationFactory opFactory = evtGen.getOperationFactory();
+ assertEquals("operation group size from the factory is not as expected: ",
+ numOpGroups, opFactory.getOperations().size());
+
+ for (int i = 0; i < numRuns; i++) {
+ int ops = numOperations;
+ loadProfile.setNumOperations(ops);
+ while (ops-- > 0) {
+ TenantOperationInfo info = evtGen.next();
+ switch (TestOperationGroup.valueOf(info.getOperationGroupId())) {
+ case upsertOp:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(UpsertOperationSupplier.class));
+ break;
+ case queryOp1:
+ case queryOp2:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(QueryOperationSupplier.class));
+ break;
+ case idleOp:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(IdleTimeOperationSupplier.class));
+ break;
+ case udfOp:
+ assertTrue(opFactory.getOperationSupplier(info).getClass()
+ .isAssignableFrom(UserDefinedOperationSupplier.class));
+ break;
+ default:
+ Assert.fail();
+
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/UniformDistributionLoadEventGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/UniformDistributionLoadEventGeneratorTest.java
new file mode 100644
index 0000000..f3c048a
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/UniformDistributionLoadEventGeneratorTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.XMLConfigParserTest;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.generators.UniformDistributionLoadEventGenerator;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the various event generation outcomes based on scenario, model and load profile.
+ */
+public class UniformDistributionLoadEventGeneratorTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(
+ UniformDistributionLoadEventGeneratorTest.class);
+
+ private enum TestOperationGroup {
+ upsertOp, queryOp1, queryOp2, queryOp3, queryOp4, queryOp5, queryOp6, queryOp7, idleOp, udfOp
+ }
+
+ private enum TestTenantGroup {
+ tg1
+ }
+
+ public DataModel readTestDataModel(String resourceName) throws Exception {
+ URL scenarioUrl = XMLConfigParserTest.class.getResource(resourceName);
+ assertNotNull(scenarioUrl);
+ Path p = Paths.get(scenarioUrl.toURI());
+ return XMLConfigParser.readDataModel(p);
+ }
+
+ /**
+ * Case : where no operations and tenant groups have zero weight
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testVariousEventGeneration() throws Exception {
+ int numRuns = 100;
+ int numOperations = 1000;
+ double normalizedOperations = (double) (numOperations * numRuns) / 10000.0f;
+ int numTenantGroups = 1;
+ int numOpGroups = 10;
+ double variancePercent = 0.05f; // 5 percent
+
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ Properties properties = PherfConstants
+ .create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+
+ DataModel model = readTestDataModel("/scenario/test_evt_gen3.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+ assertEquals("tenant group size is not as expected: ",
+ numTenantGroups, loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ",
+ numOpGroups, loadProfile.getOpDistribution().size());
+ // Calculate the expected distribution.
+ double[][] expectedDistribution = new double[numOpGroups][numTenantGroups];
+ int tenantWeight = 100;
+ int opWeight = 10;
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ expectedDistribution[r][c] = normalizedOperations * (tenantWeight * opWeight);
+ LOGGER.debug(String.format("Expected [%d,%d] = %f", r, c, expectedDistribution[r][c]));
+ }
+ }
+
+ UniformDistributionLoadEventGenerator
+ evtGen = new UniformDistributionLoadEventGenerator(
+ pUtil, model, scenario, properties);
+
+ // Calculate the actual distribution.
+ double[][] distribution = new double[numOpGroups][numTenantGroups];
+ for (int i = 0; i < numRuns; i++) {
+ int ops = numOperations;
+ loadProfile.setNumOperations(ops);
+ while (ops-- > 0) {
+ TenantOperationInfo info = evtGen.next();
+ int row = TestOperationGroup.valueOf(info.getOperationGroupId()).ordinal();
+ int col = TestTenantGroup.valueOf(info.getTenantGroupId()).ordinal();
+ distribution[row][col]++;
+ }
+ }
+
+ // Validate that the expected and actual distribution
+ // is within the margin of allowed variance.
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ double allowedVariance = expectedDistribution[r][c] * variancePercent;
+ double diff = Math.abs(expectedDistribution[r][c] - distribution[r][c]);
+ boolean isAllowed = diff < allowedVariance;
+ LOGGER.debug(String.format("Actual[%d,%d] = %f, %f, %f",
+ r, c, distribution[r][c], diff, allowedVariance));
+ assertTrue(String.format("Difference is outside the allowed variance "
+ + "[expected = %f, actual = %f]", allowedVariance, diff), isAllowed);
+
+ }
+ }
+ }
+ }
+}
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/WeightedRandomLoadEventGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/WeightedRandomLoadEventGeneratorTest.java
new file mode 100644
index 0000000..ddc01b3
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/workload/mt/WeightedRandomLoadEventGeneratorTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.phoenix.pherf.workload.mt;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.XMLConfigParserTest;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.LoadProfile;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.apache.phoenix.pherf.workload.mt.generators.TenantOperationInfo;
+import org.apache.phoenix.pherf.workload.mt.generators.WeightedRandomLoadEventGenerator;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the various event generation outcomes based on scenario, model and load profile.
+ */
+public class WeightedRandomLoadEventGeneratorTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(
+ WeightedRandomLoadEventGeneratorTest.class);
+ private enum TestOperationGroup {
+ upsertOp, queryOp1, queryOp2, idleOp, udfOp
+ }
+
+ private enum TestOperationGroup2 {
+ upsertOp, queryOp1, queryOp2, queryOp3, queryOp4, queryOp5, queryOp6, queryOp7, queryOp8, idleOp, udfOp
+ }
+
+ private enum TestTenantGroup {
+ tg1, tg2, tg3
+ }
+
+ public DataModel readTestDataModel(String resourceName) throws Exception {
+ URL scenarioUrl = XMLConfigParserTest.class.getResource(resourceName);
+ assertNotNull(scenarioUrl);
+ Path p = Paths.get(scenarioUrl.toURI());
+ return XMLConfigParser.readDataModel(p);
+ }
+
+ /**
+ * Case : where no operations and tenant groups have zero weight
+ * @throws Exception
+ */
+ @Test
+ public void testVariousEventGeneration() throws Exception {
+ int numRuns = 10;
+ int numOperations = 100000;
+ double normalizedOperations = (double) (numOperations * numRuns) / 10000.0f;
+ int numTenantGroups = 3;
+ int numOpGroups = 5;
+
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ Properties properties = PherfConstants
+ .create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+
+ DataModel model = readTestDataModel("/scenario/test_evt_gen1.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+ assertEquals("tenant group size is not as expected: ",
+ numTenantGroups, loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ",
+ numOpGroups, loadProfile.getOpDistribution().size());
+ // Calculate the expected distribution.
+ double[][] expectedDistribution = new double[numOpGroups][numTenantGroups];
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ int tenantWeight = loadProfile.getTenantDistribution().get(c).getWeight();
+ int opWeight = loadProfile.getOpDistribution().get(r).getWeight();
+ expectedDistribution[r][c] = normalizedOperations * (tenantWeight * opWeight);
+ LOGGER.debug(String.format("Expected [%d,%d] = %f", r, c, expectedDistribution[r][c]));
+ }
+ }
+
+ WeightedRandomLoadEventGenerator evtGen = new WeightedRandomLoadEventGenerator(
+ pUtil, model, scenario, properties);
+
+ // Calculate the actual distribution.
+ double[][] distribution = new double[numOpGroups][numTenantGroups];
+ for (int i = 0; i < numRuns; i++) {
+ int ops = numOperations;
+ loadProfile.setNumOperations(ops);
+ while (ops-- > 0) {
+ TenantOperationInfo info = evtGen.next();
+ int row = TestOperationGroup.valueOf(info.getOperationGroupId()).ordinal();
+ int col = TestTenantGroup.valueOf(info.getTenantGroupId()).ordinal();
+ distribution[row][col]++;
+ }
+ }
+ validateResults(numOpGroups, numTenantGroups, expectedDistribution, distribution);
+ }
+ }
+
+ /**
+ * Case : where some operations have zero weight
+ */
+ @Test
+ public void testAutoAssignedPMFs() throws Exception {
+ int numRuns = 50;
+ int numOperations = 100000;
+ double normalizedOperations = (double) (numOperations * numRuns) / 10000.0f;
+ int numTenantGroups = 3;
+ int numOpGroups = 11;
+
+ PhoenixUtil pUtil = PhoenixUtil.create();
+ Properties properties = PherfConstants
+ .create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
+
+ DataModel model = readTestDataModel("/scenario/test_evt_gen2.xml");
+ for (Scenario scenario : model.getScenarios()) {
+ LOGGER.debug(String.format("Testing %s", scenario.getName()));
+ LoadProfile loadProfile = scenario.getLoadProfile();
+ assertEquals("tenant group size is not as expected: ",
+ numTenantGroups, loadProfile.getTenantDistribution().size());
+ assertEquals("operation group size is not as expected: ",
+ numOpGroups, loadProfile.getOpDistribution().size());
+
+ float totalOperationWeight = 0.0f;
+ float autoAssignedOperationWeight = 0.0f;
+ float remainingOperationWeight = 0.0f;
+ int numAutoWeightedOperations = 0;
+ for (int r = 0; r < numOpGroups; r++) {
+ int opWeight = loadProfile.getOpDistribution().get(r).getWeight();
+ if (opWeight > 0.0f) {
+ totalOperationWeight += opWeight;
+ } else {
+ numAutoWeightedOperations++;
+ }
+ }
+ remainingOperationWeight = 100.0f - totalOperationWeight;
+ if (numAutoWeightedOperations > 0) {
+ autoAssignedOperationWeight = remainingOperationWeight/((float) numAutoWeightedOperations);
+ }
+ LOGGER.debug(String.format("Auto [%d,%f] = %f", numAutoWeightedOperations,
+ remainingOperationWeight, autoAssignedOperationWeight ));
+
+ // Calculate the expected distribution.
+ double[][] expectedDistribution = new double[numOpGroups][numTenantGroups];
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ float tenantWeight = loadProfile.getTenantDistribution().get(c).getWeight();
+ float opWeight = loadProfile.getOpDistribution().get(r).getWeight();
+ if (opWeight <= 0.0f) {
+ opWeight = autoAssignedOperationWeight;
+ }
+ expectedDistribution[r][c] = Math.round(normalizedOperations * (tenantWeight * opWeight));
+ LOGGER.debug(String.format("Expected [%d,%d] = %f", r, c, expectedDistribution[r][c]));
+ }
+ }
+
+ WeightedRandomLoadEventGenerator evtGen = new WeightedRandomLoadEventGenerator(
+ pUtil, model, scenario, properties);
+
+ // Calculate the actual distribution.
+ double[][] distribution = new double[numOpGroups][numTenantGroups];
+ for (int i = 0; i < numRuns; i++) {
+ int ops = numOperations;
+ loadProfile.setNumOperations(ops);
+ while (ops-- > 0) {
+ TenantOperationInfo info = evtGen.next();
+ int row = TestOperationGroup2.valueOf(info.getOperationGroupId()).ordinal();
+ int col = TestTenantGroup.valueOf(info.getTenantGroupId()).ordinal();
+ distribution[row][col]++;
+ }
+ }
+ validateResults(numOpGroups, numTenantGroups, expectedDistribution, distribution);
+ }
+ }
+
+ private void validateResults(int numOpGroups, int numTenantGroups,
+ double[][] expectedDistribution,
+ double[][] actualDistribution) throws Exception {
+
+ double variancePercent = 0.05f; // 5 percent
+
+ // Validate that the expected and actual distribution
+ // is within the margin of allowed variance.
+ for (int r = 0; r < numOpGroups; r++) {
+ for (int c = 0; c < numTenantGroups; c++) {
+ double allowedVariance = expectedDistribution[r][c] * variancePercent;
+ double diff = Math.abs(expectedDistribution[r][c] - actualDistribution[r][c]);
+ boolean isAllowed = diff < allowedVariance;
+ LOGGER.debug(String.format("Actual[%d,%d] = %f, %f, %f",
+ r, c, actualDistribution[r][c], diff, allowedVariance));
+ assertTrue(String.format("Difference is outside the allowed variance "
+ + "[expected = %f, actual = %f]", allowedVariance, diff), isAllowed);
+ }
+ }
+ }
+}
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_mt_schema_base_table.sql
similarity index 72%
copy from phoenix-pherf/src/test/resources/datamodel/test_schema.sql
copy to phoenix-pherf/src/test/resources/datamodel/test_mt_schema_base_table.sql
index fa9952b..184180a 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_mt_schema_base_table.sql
@@ -15,25 +15,17 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
+CREATE TABLE IF NOT EXISTS PHERF.TEST_BASE_TABLE (
TENANT_ID CHAR(15) NOT NULL,
- PARENT_ID CHAR(15) NOT NULL,
- CREATED_DATE DATE NOT NULL,
- NOW_DATE DATE,
- TS_DATE TIMESTAMP,
- PRESENT_DATE DATE,
- OTHER_ID CHAR(15),
+ IDENTIFIER CHAR(3) NOT NULL,
+ ID CHAR(15) NOT NULL,
+ CREATED_DATE DATE,
FIELD VARCHAR,
- VAR_ARRAY VARCHAR ARRAY,
- VAR_BIN VARBINARY,
- DIVISION INTEGER,
- OLDVAL_STRING VARCHAR,
- NEWVAL_STRING VARCHAR,
SOME_INT INTEGER
CONSTRAINT PK PRIMARY KEY
(
TENANT_ID,
- PARENT_ID,
- CREATED_DATE DESC
+ IDENTIFIER,
+ ID
)
) VERSIONS=1,MULTI_TENANT=true
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_mt_schema_view1.sql
similarity index 63%
copy from phoenix-pherf/src/test/resources/datamodel/test_schema.sql
copy to phoenix-pherf/src/test/resources/datamodel/test_mt_schema_view1.sql
index fa9952b..51080dd 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_mt_schema_view1.sql
@@ -15,25 +15,13 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
- TENANT_ID CHAR(15) NOT NULL,
- PARENT_ID CHAR(15) NOT NULL,
- CREATED_DATE DATE NOT NULL,
- NOW_DATE DATE,
- TS_DATE TIMESTAMP,
- PRESENT_DATE DATE,
- OTHER_ID CHAR(15),
- FIELD VARCHAR,
- VAR_ARRAY VARCHAR ARRAY,
- VAR_BIN VARBINARY,
- DIVISION INTEGER,
- OLDVAL_STRING VARCHAR,
- NEWVAL_STRING VARCHAR,
- SOME_INT INTEGER
+
+CREATE VIEW IF NOT EXISTS PHERF.TEST_GLOBAL_VIEW1 (
+ GID CHAR(15) NOT NULL,
+ FIELD1 VARCHAR,
+ OTHER_INT INTEGER
CONSTRAINT PK PRIMARY KEY
(
- TENANT_ID,
- PARENT_ID,
- CREATED_DATE DESC
+ GID
)
-) VERSIONS=1,MULTI_TENANT=true
+) AS SELECT * FROM PHERF.TEST_BASE_TABLE WHERE IDENTIFIER = 'EV1'
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_mt_schema_view2.sql
similarity index 63%
copy from phoenix-pherf/src/test/resources/datamodel/test_schema.sql
copy to phoenix-pherf/src/test/resources/datamodel/test_mt_schema_view2.sql
index fa9952b..a435eb7 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_mt_schema_view2.sql
@@ -15,25 +15,13 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
- TENANT_ID CHAR(15) NOT NULL,
- PARENT_ID CHAR(15) NOT NULL,
- CREATED_DATE DATE NOT NULL,
- NOW_DATE DATE,
- TS_DATE TIMESTAMP,
- PRESENT_DATE DATE,
- OTHER_ID CHAR(15),
- FIELD VARCHAR,
- VAR_ARRAY VARCHAR ARRAY,
- VAR_BIN VARBINARY,
- DIVISION INTEGER,
- OLDVAL_STRING VARCHAR,
- NEWVAL_STRING VARCHAR,
- SOME_INT INTEGER
+
+CREATE VIEW IF NOT EXISTS PHERF.TEST_GLOBAL_VIEW2 (
+ GID CHAR(15) NOT NULL,
+ FIELD1 VARCHAR,
+ OTHER_INT INTEGER
CONSTRAINT PK PRIMARY KEY
(
- TENANT_ID,
- PARENT_ID,
- CREATED_DATE DESC
+ GID
)
-) VERSIONS=1,MULTI_TENANT=true
+) AS SELECT * FROM PHERF.TEST_BASE_TABLE WHERE IDENTIFIER = 'EV2'
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
index fa9952b..9a1c856 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
@@ -19,7 +19,8 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
TENANT_ID CHAR(15) NOT NULL,
PARENT_ID CHAR(15) NOT NULL,
CREATED_DATE DATE NOT NULL,
- NOW_DATE DATE,
+ SOME_DATE DATE,
+ RND_DATE DATE,
TS_DATE TIMESTAMP,
PRESENT_DATE DATE,
OTHER_ID CHAR(15),
@@ -29,6 +30,7 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
DIVISION INTEGER,
OLDVAL_STRING VARCHAR,
NEWVAL_STRING VARCHAR,
+ CONNECTION_ID VARCHAR,
SOME_INT INTEGER
CONSTRAINT PK PRIMARY KEY
(
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_tbl_schema_simple.sql
similarity index 63%
copy from phoenix-pherf/src/test/resources/datamodel/test_schema.sql
copy to phoenix-pherf/src/test/resources/datamodel/test_tbl_schema_simple.sql
index fa9952b..8b76820 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_tbl_schema_simple.sql
@@ -15,25 +15,19 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
*/
-CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
- TENANT_ID CHAR(15) NOT NULL,
- PARENT_ID CHAR(15) NOT NULL,
- CREATED_DATE DATE NOT NULL,
- NOW_DATE DATE,
- TS_DATE TIMESTAMP,
- PRESENT_DATE DATE,
- OTHER_ID CHAR(15),
- FIELD VARCHAR,
- VAR_ARRAY VARCHAR ARRAY,
- VAR_BIN VARBINARY,
- DIVISION INTEGER,
- OLDVAL_STRING VARCHAR,
- NEWVAL_STRING VARCHAR,
- SOME_INT INTEGER
+CREATE TABLE IF NOT EXISTS PHERF.TEST_MULTI_TENANT_TABLE (
+ HOST CHAR(2) NOT NULL,
+ DOMAIN VARCHAR NOT NULL,
+ FEATURE VARCHAR NOT NULL,
+ DATE DATE NOT NULL,
+ USAGE.CORE BIGINT,
+ USAGE.DB BIGINT,
+ STATS.ACTIVE_VISITOR INTEGER
CONSTRAINT PK PRIMARY KEY
(
- TENANT_ID,
- PARENT_ID,
- CREATED_DATE DESC
+ HOST,
+ DOMAIN,
+ FEATURE,
+ DATE
)
-) VERSIONS=1,MULTI_TENANT=true
+) VERSIONS=1, MULTI_TENANT=true
diff --git a/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml b/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
index e8528f3..cc45395 100644
--- a/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
+++ b/phoenix-pherf/src/test/resources/scenario/malicious_scenario_with_dtd.xml
@@ -40,7 +40,7 @@
least once.
-->
<scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno"
- ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE"
+ ddl="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE"
rowCount="100" name="testMTDdlWriteScenario">
</scenario>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_evt_gen1.xml b/phoenix-pherf/src/test/resources/scenario/test_evt_gen1.xml
new file mode 100644
index 0000000..d8902bb
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/scenario/test_evt_gen1.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<datamodel name="model_1">
+ <datamapping>
+ <column>
+ <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
+ <type>VARCHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_VARCHAR</name>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.EVT_GEN1" name="EVT_GEN1" generatorName="WEIGHTED">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <!-- Case 1 : where no operations have zero weight -->
+ <tenantDistribution id="tg1" weight="10" numTenants="10"></tenantDistribution>
+ <tenantDistribution id="tg2" weight="10" numTenants="10"></tenantDistribution>
+ <tenantDistribution id="tg3" weight="80" numTenants="10"></tenantDistribution>
+ <opDistribution id="upsertOp" weight="40"></opDistribution>
+ <opDistribution id="queryOp1" weight="20"></opDistribution>
+ <opDistribution id="queryOp2" weight="20"></opDistribution>
+ <opDistribution id="idleOp" weight="10"></opDistribution>
+ <opDistribution id="udfOp" weight="10"></opDistribution>
+ </loadProfile>
+ <upserts>
+ <upsert id="upsertOp">
+ <column>
+ <type>CHAR</type>
+ <name>COLUMN1</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <querySet>
+ <query id="queryOp1" statement="select count(*) from PHERF.EVT_GEN1"/>
+ <query id="queryOp2" statement="select sum(SOME_INT) from PHERF.EVT_GEN1"/>
+ </querySet>
+ <idleTimes>
+ <idleTime id="idleOp" idleTime="50"></idleTime>
+ </idleTimes>
+ <udfs>
+ <udf id="udfOp" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_evt_gen2.xml b/phoenix-pherf/src/test/resources/scenario/test_evt_gen2.xml
new file mode 100644
index 0000000..4278abd
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/scenario/test_evt_gen2.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<datamodel name="model_1">
+ <datamapping>
+ <column>
+ <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
+ <type>VARCHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_VARCHAR</name>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.EVT_GEN2" name="EVT_GEN2" generatorName="WEIGHTED">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <!-- Case 1 : where some operations have zero weight -->
+ <tenantDistribution id="tg1" weight="10" numTenants="10"></tenantDistribution>
+ <tenantDistribution id="tg2" weight="10" numTenants="10"></tenantDistribution>
+ <tenantDistribution id="tg3" weight="80" numTenants="10"></tenantDistribution>
+ <opDistribution id="upsertOp" weight="40"></opDistribution>
+ <opDistribution id="queryOp1" weight="10"></opDistribution>
+ <opDistribution id="queryOp2" weight="10"></opDistribution>
+ <opDistribution id="queryOp3"></opDistribution>
+ <opDistribution id="queryOp4"></opDistribution>
+ <opDistribution id="queryOp5"></opDistribution>
+ <opDistribution id="queryOp6"></opDistribution>
+ <opDistribution id="queryOp7"></opDistribution>
+ <opDistribution id="queryOp8"></opDistribution>
+ <opDistribution id="idleOp" weight="10"></opDistribution>
+ <opDistribution id="udfOp" weight="10"></opDistribution>
+ </loadProfile>
+ <upserts>
+ <upsert id="upsertOp">
+ <column>
+ <type>CHAR</type>
+ <name>COLUMN1</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <querySet>
+ <query id="queryOp1" statement="select count(*) from PHERF.EVT_GEN2"/>
+ <query id="queryOp2" statement="select sum(SOME_INT) from PHERF.EVT_GEN2"/>
+ <query id="queryOp3" statement="select sum(SOME_INT) +3 from PHERF.EVT_GEN2"/>
+ <query id="queryOp4" statement="select sum(SOME_INT) +4 from PHERF.EVT_GEN2"/>
+ <query id="queryOp5" statement="select sum(SOME_INT) +5 from PHERF.EVT_GEN2"/>
+ <query id="queryOp6" statement="select sum(SOME_INT) +6 from PHERF.EVT_GEN2"/>
+ <query id="queryOp7" statement="select sum(SOME_INT) +7 from PHERF.EVT_GEN2"/>
+ <query id="queryOp8" statement="select sum(SOME_INT) +8 from PHERF.EVT_GEN2"/>
+ </querySet>
+ <idleTimes>
+ <idleTime id="idleOp" idleTime="50"></idleTime>
+ </idleTimes>
+ <udfs>
+ <udf id="udfOp" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_evt_gen3.xml b/phoenix-pherf/src/test/resources/scenario/test_evt_gen3.xml
new file mode 100644
index 0000000..eb55f5d
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/scenario/test_evt_gen3.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<datamodel name="model_1">
+ <datamapping>
+ <column>
+ <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
+ <type>VARCHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_VARCHAR</name>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.EVT_GEN3" name="EVT_GEN3" generatorName="UNIFORM">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <!-- Case 1 : where some operations have zero weight -->
+ <tenantDistribution id="tg1" numTenants="1"></tenantDistribution>
+ <opDistribution id="upsertOp"></opDistribution>
+ <opDistribution id="queryOp1"></opDistribution>
+ <opDistribution id="queryOp2"></opDistribution>
+ <opDistribution id="queryOp3"></opDistribution>
+ <opDistribution id="queryOp4"></opDistribution>
+ <opDistribution id="queryOp5"></opDistribution>
+ <opDistribution id="queryOp6"></opDistribution>
+ <opDistribution id="queryOp7"></opDistribution>
+ <opDistribution id="idleOp"></opDistribution>
+ <opDistribution id="udfOp"></opDistribution>
+ </loadProfile>
+ <upserts>
+ <upsert id="upsertOp">
+ <column>
+ <type>CHAR</type>
+ <name>COLUMN1</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <querySet>
+ <query id="queryOp1" statement="select count(*) from PHERF.EVT_GEN3"/>
+ <query id="queryOp2" statement="select sum(SOME_INT) from PHERF.EVT_GEN3"/>
+ <query id="queryOp3" statement="select sum(SOME_INT) +3 from PHERF.EVT_GEN3"/>
+ <query id="queryOp4" statement="select sum(SOME_INT) +4 from PHERF.EVT_GEN3"/>
+ <query id="queryOp5" statement="select sum(SOME_INT) +5 from PHERF.EVT_GEN3"/>
+ <query id="queryOp6" statement="select sum(SOME_INT) +6 from PHERF.EVT_GEN3"/>
+ <query id="queryOp7" statement="select sum(SOME_INT) +7 from PHERF.EVT_GEN3"/>
+ </querySet>
+ <idleTimes>
+ <idleTime id="idleOp" idleTime="50"></idleTime>
+ </idleTimes>
+ <udfs>
+ <udf id="udfOp" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_evt_gen4.xml b/phoenix-pherf/src/test/resources/scenario/test_evt_gen4.xml
new file mode 100644
index 0000000..d947e90
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/scenario/test_evt_gen4.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<datamodel name="model_1">
+ <datamapping>
+ <column>
+ <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
+ <type>VARCHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_VARCHAR</name>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.EVT_GEN4" name="EVT_GEN4" generatorName="SEQUENTIAL">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <!-- Case 1 : where some operations have zero weight -->
+ <tenantDistribution id="tg1" numTenants="1"></tenantDistribution>
+ <opDistribution id="upsertOp"></opDistribution>
+ <opDistribution id="queryOp1"></opDistribution>
+ <opDistribution id="queryOp2"></opDistribution>
+ <opDistribution id="queryOp3"></opDistribution>
+ <opDistribution id="queryOp4"></opDistribution>
+ <opDistribution id="queryOp5"></opDistribution>
+ <opDistribution id="queryOp6"></opDistribution>
+ <opDistribution id="queryOp7"></opDistribution>
+ <opDistribution id="idleOp"></opDistribution>
+ <opDistribution id="udfOp"></opDistribution>
+ </loadProfile>
+ <upserts>
+ <upsert id="upsertOp">
+ <column>
+ <type>CHAR</type>
+ <name>COLUMN1</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <querySet>
+ <query id="queryOp1" statement="select count(*) from PHERF.EVT_GEN3"/>
+ <query id="queryOp2" statement="select sum(SOME_INT) from PHERF.EVT_GEN3"/>
+ <query id="queryOp3" statement="select sum(SOME_INT) +3 from PHERF.EVT_GEN3"/>
+ <query id="queryOp4" statement="select sum(SOME_INT) +4 from PHERF.EVT_GEN3"/>
+ <query id="queryOp5" statement="select sum(SOME_INT) +5 from PHERF.EVT_GEN3"/>
+ <query id="queryOp6" statement="select sum(SOME_INT) +6 from PHERF.EVT_GEN3"/>
+ <query id="queryOp7" statement="select sum(SOME_INT) +7 from PHERF.EVT_GEN3"/>
+ </querySet>
+ <idleTimes>
+ <idleTime id="idleOp" idleTime="50"></idleTime>
+ </idleTimes>
+ <udfs>
+ <udf id="udfOp" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_mt_workload_template.xml b/phoenix-pherf/src/test/resources/scenario/test_mt_workload_template.xml
new file mode 100644
index 0000000..01d066d
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/scenario/test_mt_workload_template.xml
@@ -0,0 +1,226 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+ <datamodel name="model_1">
+ <datamapping>
+ <column>
+ <!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
+ <type>VARCHAR</type>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_VARCHAR</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <userDefined>true</userDefined>
+ <dataSequence>RANDOM</dataSequence>
+ <length>15</length>
+ <name>GENERAL_CHAR</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <dataSequence>RANDOM</dataSequence>
+ <minValue>1</minValue>
+ <maxValue>50000000</maxValue>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <name>GENERAL_INTEGER</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <name>INT_ID</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <!--SEQUENTIAL is unsupported for DATE -->
+ <dataSequence>RANDOM</dataSequence>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <useCurrentDate>true</useCurrentDate>
+ <name>GENERAL_DATE</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <length>3</length>
+ <userDefined>true</userDefined>
+ <dataSequence>LIST</dataSequence>
+ <name>TYPE</name>
+ <valuelist>
+ <!-- Distributes according to specified values. These must total 100 -->
+ <datavalue distribution="60">
+ <value>ABC</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>XYZ</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>LMN</value>
+ </datavalue>
+ </valuelist>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <length>3</length>
+ <name>IDENTIFIER</name>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.EVT_1" name="EVT_1" generatorName="WEIGHTED">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>10000</numOperations>
+ <!-- Case 1 : Upsert Operation test -->
+ <tenantDistribution id="tg1" weight="10" numTenants="1"/>
+ <tenantDistribution id="tg2" weight="40" numTenants="9"/>
+ <tenantDistribution id="tg3" weight="50" numTenants="10"/>
+ <opDistribution id="upsertOp" weight="20"/>
+ <opDistribution id="queryOp1" weight="20"/>
+ <opDistribution id="queryOp2" weight="20"/>
+ <opDistribution id="idleOp" weight="20"/>
+ <opDistribution id="udfOp" weight="20"/>
+ </loadProfile>
+ <preScenarioDdls>
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.EVT_1 (ZID CHAR(15), TYPE VARCHAR) AS SELECT * FROM PHERF.TEST_GLOBAL_VIEW1" />
+ </preScenarioDdls>
+
+ <upserts>
+ <upsert id="upsertOp">
+ <column>
+ <type>CHAR</type>
+ <name>ID</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <name>SOME_INT</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <name>GID</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD1</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <name>OTHER_INT</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <name>ZID</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <name>TYPE</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <querySet>
+ <query id="queryOp1" statement="select count(*) from PHERF.EVT_1"/>
+ <query id="queryOp2" statement="select * from PHERF.EVT_1"/>
+ </querySet>
+ <idleTimes>
+ <idleTime id="idleOp" idleTime="50"></idleTime>
+ </idleTimes>
+ <udfs>
+ <udf id="udfOp" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ </scenario>
+ <scenario tableName="PHERF.EVT_2" name="EVT_21" generatorName="WEIGHTED">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>10000</numOperations>
+ <!-- Case 1 : Upsert Operation test -->
+ <tenantDistribution id="tg1" weight="10" numTenants="1"/>
+ <tenantDistribution id="tg2" weight="40" numTenants="9"/>
+ <tenantDistribution id="tg3" weight="50" numTenants="10"/>
+ <opDistribution id="upsertOp" weight="20"/>
+ <opDistribution id="queryOp1" weight="20"/>
+ <opDistribution id="queryOp2" weight="20"/>
+ <opDistribution id="idleOp" weight="20"/>
+ <opDistribution id="udfOp" weight="20"/>
+ </loadProfile>
+ <preScenarioDdls>
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.EVT_2 (INT_ID INTEGER, TYPE VARCHAR) AS SELECT * FROM PHERF.TEST_GLOBAL_VIEW2" />
+ </preScenarioDdls>
+
+ <upserts>
+ <upsert id="upsertOp">
+ <column>
+ <type>CHAR</type>
+ <name>ID</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <name>SOME_INT</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <name>GID</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD1</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <name>OTHER_INT</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <name>INT_ID</name>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <name>TYPE</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <querySet>
+ <query id="queryOp1" statement="select count(*) from PHERF.EVT_2"/>
+ <query id="queryOp2" statement="select * from PHERF.EVT_2"/>
+ </querySet>
+ <idleTimes>
+ <idleTime id="idleOp" idleTime="50"/>
+ </idleTimes>
+ <udfs>
+ <udf id="udfOp" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
index 076f236..db3caac 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
@@ -59,13 +59,14 @@
<column>
<type>DATE</type>
<!--SEQUENTIAL is unsupported for DATE -->
+ <userDefined>true</userDefined>
<dataSequence>RANDOM</dataSequence>
<!-- Number [0-100] that represents the probability of creating a null value -->
<!-- The higher the number, the more like the value will returned will be null -->
<!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
<nullChance>0</nullChance>
<useCurrentDate>true</useCurrentDate>
- <name>NOW_DATE</name>
+ <name>RND_DATE</name>
</column>
<column>
<type>DECIMAL</type>
@@ -94,7 +95,20 @@
</column>
<column>
<type>DATE</type>
+ <userDefined>true</userDefined>
+ <!--SEQUENTIAL is now supported for DATE -->
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <useCurrentDate>true</useCurrentDate>
<name>CREATED_DATE</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <userDefined>true</userDefined>
+ <name>SOME_DATE</name>
<minValue>1975</minValue>
<maxValue>2025</maxValue>
<valuelist>
@@ -115,6 +129,7 @@
</column>
<column>
<type>DATE</type>
+ <userDefined>true</userDefined>
<name>PRESENT_DATE</name>
<minValue>1975</minValue>
<maxValue>2025</maxValue>
@@ -262,7 +277,7 @@
-->
<threadSleepDuration>10</threadSleepDuration>
- <batchSize>1000</batchSize>
+ <batchSize>1</batchSize>
</writeParams>
<querySet concurrency="1" executionType="PARALLEL" executionDurationInMs="10000">
<query id="q3" statement="select count(*) from PHERF.TEST_TABLE"/>
@@ -306,12 +321,12 @@
<scenario tableName="PHERF.TEST_TABLE" rowCount="99" name="testPreAndPostDdls">
<preScenarioDdls>
- <ddl statement="CREATE INDEX IDX_DIVISION ON ? (DIVISION)" tableName="PHERF.PHERF_PROD_TEST_UNSALTED"/>
+ <ddl statement="CREATE INDEX IDX_DIVISION ON ? (DIVISION)" tableName="PHERF.TEST_TABLE"/>
</preScenarioDdls>
<postScenarioDdls>
- <ddl statement="CREATE INDEX IDX_OLDVAL_STRING ON ? (OLDVAL_STRING)" tableName="PHERF.PHERF_PROD_TEST_UNSALTED"/>
- <ddl statement="CREATE INDEX IDX_CONNECTION_ID ON ? (CONNECTION_ID)" tableName="PHERF.PHERF_PROD_TEST_UNSALTED"/>
+ <ddl statement="CREATE INDEX IDX_OLDVAL_STRING ON ? (OLDVAL_STRING)" tableName="PHERF.TEST_TABLE"/>
+ <ddl statement="CREATE INDEX IDX_CONNECTION_ID ON ? (CONNECTION_ID)" tableName="PHERF.TEST_TABLE"/>
</postScenarioDdls>
<querySet concurrency="1" executionType="SERIAL" executionDurationInMs="5000"
@@ -331,7 +346,7 @@
<scenario tableName="PHERF.TEST_VIEW" tenantId="xyzdefghijklmno"
rowCount="100" name="testMTWriteScenario">
<preScenarioDdls>
- <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" />
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" />
</preScenarioDdls>
</scenario>
<!-- Scenario level DDL that is dynamically executed before the Write Workload is run.
@@ -344,7 +359,7 @@
<scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno"
rowCount="100" name="testMTDdlWriteScenario">
<preScenarioDdls>
- <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" />
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" />
</preScenarioDdls>
</scenario>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_tbl_workload_template.xml b/phoenix-pherf/src/test/resources/scenario/test_tbl_workload_template.xml
new file mode 100644
index 0000000..32c79de
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/scenario/test_tbl_workload_template.xml
@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<datamodel name="TEST_MT_TABLE_MODEL">
+ <datamapping>
+ <column>
+ <type>CHAR</type>
+ <length>2</length>
+ <userDefined>true</userDefined>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <name>HOST</name>
+ <valuelist>
+ <!-- Distributes according to specified values. These must total 100 -->
+ <!-- ["NA","CS","EU"] -->
+ <datavalue distribution="34">
+ <value>NA</value>
+ </datavalue>
+ <datavalue distribution="33">
+ <value>CS</value>
+ </datavalue>
+ <datavalue distribution="33">
+ <value>EU</value>
+ </datavalue>
+ </valuelist>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <userDefined>true</userDefined>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <name>DOMAIN</name>
+ <valuelist>
+ <!-- Distributes according to specified values. These must total 100 -->
+ <!-- ["Salesforce.com","Apple.com","Google.com","Amazon.com"]-->
+ <datavalue distribution="25">
+ <value>Salesforce.com</value>
+ </datavalue>
+ <datavalue distribution="25">
+ <value>Apple.com</value>
+ </datavalue>
+ <datavalue distribution="25">
+ <value>Google.com</value>
+ </datavalue>
+ <datavalue distribution="25">
+ <value>Amazon.com</value>
+ </datavalue>
+ </valuelist>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <userDefined>true</userDefined>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <name>FEATURE</name>
+ <valuelist>
+ <!-- Distributes according to specified values. These must total 100 -->
+ <!-- ["Login","Report","Dashboard","Sales","UI"]-->
+ <datavalue distribution="20">
+ <value>Login</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>Report</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>Dashboard</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>Sales</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>UI</value>
+ </datavalue>
+ </valuelist>
+ </column>
+ <column>
+ <type>DATE</type>
+ <userDefined>true</userDefined>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <name>DATE</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <userDefined>true</userDefined>
+ <!--SEQUENTIAL is unsupported for DATE -->
+ <dataSequence>RANDOM</dataSequence>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <minValue>2020</minValue>
+ <maxValue>2025</maxValue>
+ <name>DATE_O</name>
+ </column>
+
+ <column>
+ <type>BIGINT</type>
+ <dataSequence>RANDOM</dataSequence>
+ <minValue>1</minValue>
+ <maxValue>100</maxValue>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <name>USAGE.CORE</name>
+ </column>
+ <column>
+ <type>BIGINT</type>
+ <dataSequence>RANDOM</dataSequence>
+ <minValue>1</minValue>
+ <maxValue>2000</maxValue>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <name>USAGE.DB</name>
+ </column>
+ <column>
+ <type>INTEGER</type>
+ <dataSequence>RANDOM</dataSequence>
+ <minValue>1</minValue>
+ <maxValue>10000</maxValue>
+ <!-- Number [0-100] that represents the probability of creating a null value -->
+ <!-- The higher the number, the more like the value will returned will be null -->
+ <!-- Leaving this tag out is equivalent to having a 0 probability. i.e. never null -->
+ <nullChance>0</nullChance>
+ <name>STATS.ACTIVE_VISITOR</name>
+ </column>
+ </datamapping>
+ <scenarios>
+ <scenario tableName="PHERF.TEST_MULTI_TENANT_TABLE" name="TEST_TABLE_WRITE" generatorName="WEIGHTED">
+ <loadProfile>
+ <numOperations>1</numOperations>
+ <tenantDistribution useGlobalConnection="true"/>
+ <opDistribution id="upsertOp" weight="100"/>
+ </loadProfile>
+
+ <upserts>
+ <upsert id="upsertOp" useGlobalConnection="true" upsertGroup="write"/>
+ </upserts>
+ </scenario>
+ <scenario tableName="PHERF.TEST_MULTI_TENANT_TABLE" name="TEST_TABLE_READ" generatorName="WEIGHTED">
+ <loadProfile>
+ <numOperations>1</numOperations>
+ <tenantDistribution useGlobalConnection="true"/>
+ <opDistribution id="queryOp1" weight="50"/>
+ <opDistribution id="queryOp2" weight="50"/>
+ </loadProfile>
+
+ <querySet>
+ <query id="queryOp1" useGlobalConnection="true" statement="select count(*) from PHERF.TEST_MULTI_TENANT_TABLE" queryGroup="Aggregation"/>
+ <query id="queryOp2" useGlobalConnection="true" statement="select * from PHERF.TEST_MULTI_TENANT_TABLE LIMIT 100" queryGroup="LIMIT"/>
+ </querySet>
+ </scenario>
+ </scenarios>
+</datamodel>
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
similarity index 57%
copy from phoenix-pherf/src/test/resources/scenario/test_scenario.xml
copy to phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
index 076f236..c75e8d0 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_workload_with_load_profile.xml
@@ -17,7 +17,7 @@
~ limitations under the License.
-->
-<datamodel name="test_scenario">
+<datamodel name="model_1">
<datamapping>
<column>
<!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
@@ -196,7 +196,7 @@
<datavalue distribution="20">
<value>LMN</value>
</datavalue>
- </valuelist>
+ </valuelist>
</column>
<column>
<type>CHAR</type>
@@ -215,12 +215,12 @@
<prefix>VBOxx00</prefix>
</column>
<column>
- <type>VARCHAR</type>
- <userDefined>true</userDefined>
- <dataSequence>SEQUENTIAL</dataSequence>
- <length>1</length>
- <name>FIELD</name>
- </column>
+ <type>VARCHAR</type>
+ <userDefined>true</userDefined>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <length>1</length>
+ <name>FIELD</name>
+ </column>
<column>
<type>INTEGER</type>
<dataSequence>SEQUENTIAL</dataSequence>
@@ -230,123 +230,196 @@
</column>
</datamapping>
<scenarios>
- <scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW">
- <!-- Scenario level rule overrides will be unsupported in V1.
- You can use the general datamappings in the mean time-->
- <dataOverride>
- <column>
- <type>VARCHAR</type>
- <userDefined>true</userDefined>
- <dataSequence>RANDOM</dataSequence>
- <length>5</length>
- <name>FIELD</name>
- </column>
- </dataOverride>
+ <scenario tableName="PHERF.Z11" name="scenario_11" generatorName="UNIFORM">
+ <phoenixProperties>
+ <entry>
+ <key>pherf.mt.handlers_per_scenario</key>
+ <value>2</value>
+ </entry>
+ <entry>
+ <key>pherf.mt.uniform.benchmark_mode</key>
+ <value>true</value>
+ </entry>
+ </phoenixProperties>
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <tenantDistribution id="t111" weight="10" numTenants="10"/>
+ <tenantDistribution id="t112" weight="10" numTenants="10"/>
+ <tenantDistribution id="t113" weight="80" numTenants="1"/>
+ <opDistribution id="op111" weight="50"/>
+ <opDistribution id="op112" weight="0"/>
+ <opDistribution id="op113" weight="0"/>
+ <opDistribution id="op114" weight="50"/>
+ <opDistribution id="op115" weight="0"/>
+ </loadProfile>
- <!--
- This is used to add mixed R/W workloads.
- If this tag exists, a writer pool will be created based on the below properties.
- These props will override the default values in pherf.properties, but only for this
- scenario.The write jobs will run in conjunction with the querySet below.
- -->
- <writeParams executionDurationInMs="10000">
- <!--
- Number of writer it insert into the threadpool
- -->
- <writerThreadCount>2</writerThreadCount>
+ <preScenarioDdls>
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.Z11 (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" />
+ </preScenarioDdls>
- <!--
- Time in Ms that each thread will sleep between batch writes. This helps to
- throttle writers.
- -->
- <threadSleepDuration>10</threadSleepDuration>
+ <upserts>
+ <upsert id="op111">
+ <column>
+ <type>CHAR</type>
+ <name>PARENT_ID</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <name>CREATED_DATE</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OTHER_ID</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OLDVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>NEWVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD1</name>
+ </column>
+ </upsert>
+ </upserts>
- <batchSize>1000</batchSize>
- </writeParams>
- <querySet concurrency="1" executionType="PARALLEL" executionDurationInMs="10000">
- <query id="q3" statement="select count(*) from PHERF.TEST_TABLE"/>
- <query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/>
+ <idleTimes>
+ <idleTime id="op114" idleTime="50"/>
+ </idleTimes>
+ <udfs>
+ <udf id="op115" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+ <querySet>
+ <query id="op112" statement="select count(*) from PHERF.Z11"/>
+ <query id="op113" statement="select sum(SOME_INT) from PHERF.Z11"/>
</querySet>
</scenario>
+ <scenario tableName="PHERF.Z12" name="scenario_12">
+ <loadProfile>
+ <batchSize>5</batchSize>
+ <numOperations>1000</numOperations>
+ <tenantDistribution id="t121" weight="10" numTenants="5"/>
+ <tenantDistribution id="t122" weight="10" numTenants="5"/>
+ <tenantDistribution id="t123" weight="80" numTenants="5"/>
+ <opDistribution id="op121" weight="50"/>
+ <opDistribution id="op122" weight="5"/>
+ <opDistribution id="op123" weight="5"/>
+ <opDistribution id="op124" weight="40"/>
+ <opDistribution id="op125" weight="0"/>
+ </loadProfile>
- <scenario tableName="PHERF.TEST_TABLE" rowCount="30" name="testScenario">
- <!-- Scenario level rule overrides will be unsupported in V1.
- You can use the general datamappings in the mean time-->
- <dataOverride>
- <column>
- <type>VARCHAR</type>
- <userDefined>true</userDefined>
- <dataSequence>RANDOM</dataSequence>
- <length>10</length>
- <name>FIELD</name>
- </column>
- </dataOverride>
-
- <!--Note: 1. Minimum of executionDurationInMs or numberOfExecutions. Which ever is reached first
- 2. DDL included in query are executed only once on start of querySet execution.
- -->
- <querySet concurrency="1-3" executionType="SERIAL" executionDurationInMs="5000"
- numberOfExecutions="100">
- <query id="q1" tenantId="123456789012345" expectedAggregateRowCount="0"
- statement="select count(*) from PHERF.TEST_TABLE"/>
- <!-- queryGroup is a way to organize queries across tables or scenario files.
- The value will be dumped to results. This gives a value to group by on reporting to compare queries -->
- <query id="q2" queryGroup="g1"
- statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/>
- </querySet>
- <!--Minimum of executionDurationInMs or numberOfExecutions. Which ever is reached first -->
- <querySet concurrency="2-3" executionType="PARALLEL" executionDurationInMs="10000"
- numberOfExecutions="10">
- <query id="q3" statement="select count(*) from PHERF.TEST_TABLE"/>
- <query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/>
- </querySet>
- </scenario>
-
- <scenario tableName="PHERF.TEST_TABLE" rowCount="99" name="testPreAndPostDdls">
<preScenarioDdls>
- <ddl statement="CREATE INDEX IDX_DIVISION ON ? (DIVISION)" tableName="PHERF.PHERF_PROD_TEST_UNSALTED"/>
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.Z12 (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE" />
</preScenarioDdls>
-
- <postScenarioDdls>
- <ddl statement="CREATE INDEX IDX_OLDVAL_STRING ON ? (OLDVAL_STRING)" tableName="PHERF.PHERF_PROD_TEST_UNSALTED"/>
- <ddl statement="CREATE INDEX IDX_CONNECTION_ID ON ? (CONNECTION_ID)" tableName="PHERF.PHERF_PROD_TEST_UNSALTED"/>
- </postScenarioDdls>
-
- <querySet concurrency="1" executionType="SERIAL" executionDurationInMs="5000"
- numberOfExecutions="1">
- <query id="q1" expectedAggregateRowCount="99" statement="select count(*) from PHERF.TEST_TABLE"/>
+
+ <upserts>
+ <upsert id="op121">
+ <column>
+ <type>CHAR</type>
+ <name>PARENT_ID</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <name>CREATED_DATE</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OTHER_ID</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OLDVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>NEWVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD1</name>
+ </column>
+ </upsert>
+ </upserts>
+
+ <idleTimes>
+ <idleTime id="op124" idleTime="100"/>
+ </idleTimes>
+ <querySet>
+ <query id="op122" statement="select count(*) from PHERF.Z12"/>
+ <query id="op123" statement="select sum(SOME_INT) from PHERF.Z12"/>
</querySet>
+ <udfs>
+ <udf id="op125" >
+ <clazzName>org.apache.phoenix.pherf.ConfigurationParserTest.TestUDF</clazzName>
+ <args>Hello</args>
+ <args>World</args>
+ </udf>
+ </udfs>
+
</scenario>
-
- <!-- To configure a Write Workload to write to a tenant specific view users need to
- specify the tenantId attribute on the scenario, specifying the tenant they
- want to write data for as the attribute value. This tells Pherf to take out a
- tenant-specific connection for executing the write workload.
- The name of the tenant specific view to write to can then be specified as the value of
- the tablename attribute. This assumes the tenant specific view has been created. To
- dynamically create the view see comments below with regard to the ddl attribute.
- -->
- <scenario tableName="PHERF.TEST_VIEW" tenantId="xyzdefghijklmno"
- rowCount="100" name="testMTWriteScenario">
- <preScenarioDdls>
- <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" />
- </preScenarioDdls>
- </scenario>
- <!-- Scenario level DDL that is dynamically executed before the Write Workload is run.
- This pattern is really useful when you want to write data to multi-tenant view and the tenant id is
- tightly bound to the scenario. In such cases you can't create the view through the data model flow.
- The value of the tableName attribute is name of the view that is dynamically created based on the DDL
- in the ddl attribute. Queries accessing the View will need to manually make sure Pherf was run with the -l option at
- least once.
- -->
- <scenario tableName="PHERF.TEST_MT_VIEW" tenantId="abcdefghijklmno"
- rowCount="100" name="testMTDdlWriteScenario">
+ <scenario tableName="PHERF.Z13" name="scenario_13">
+ <loadProfile>
+ <batchSize>1</batchSize>
+ <numOperations>1000</numOperations>
+ <tenantDistribution useGlobalConnection="true"/>
+ <opDistribution id="op131" weight="100"/>
+ </loadProfile>
+
+
<preScenarioDdls>
- <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.TEST_MT_VIEW (field1 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE" />
+ <ddl statement="CREATE VIEW IF NOT EXISTS PHERF.Z13 (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_TABLE WHERE PARENT_ID = 'aAAyYhnNbBs9kWk'" />
</preScenarioDdls>
+
+ <upserts>
+ <upsert id="op131" useGlobalConnection="true">
+ <column>
+ <type>CHAR</type>
+ <name>PARENT_ID_SEQ</name>
+ </column>
+ <column>
+ <type>DATE</type>
+ <name>CREATED_DATE</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OTHER_ID</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>OLDVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>NEWVAL_STRING</name>
+ </column>
+ <column>
+ <type>VARCHAR</type>
+ <name>FIELD1</name>
+ </column>
+ </upsert>
+ </upserts>
</scenario>
-
</scenarios>
</datamodel>