You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by co...@apache.org on 2015/10/20 22:55:31 UTC
phoenix git commit: PHOENIX-2337 - Pherf - Prefix values override
rule values for writes
Repository: phoenix
Updated Branches:
refs/heads/master 28007f804 -> caa0dbb46
PHOENIX-2337 - Pherf - Prefix values override rule values for writes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/caa0dbb4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/caa0dbb4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/caa0dbb4
Branch: refs/heads/master
Commit: caa0dbb46a47945eaa384b31f7a2ba8e9c7a6c08
Parents: 28007f8
Author: Cody Marcel <co...@gmail.com>
Authored: Mon Oct 19 13:25:15 2015 -0700
Committer: Cody Marcel <cm...@cmarcel-wsl1.internal.salesforce.com>
Committed: Tue Oct 20 13:55:12 2015 -0700
----------------------------------------------------------------------
.../phoenix/pherf/rules/RulesApplier.java | 70 ++++++++++++++------
.../apache/phoenix/pherf/util/PhoenixUtil.java | 23 +++----
.../apache/phoenix/pherf/RuleGeneratorTest.java | 51 ++++++++++++--
.../test/resources/datamodel/test_schema.sql | 1 +
.../test/resources/scenario/test_scenario.xml | 14 +++-
5 files changed, 116 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
index 4801081..06a046d 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
@@ -19,6 +19,7 @@
package org.apache.phoenix.pherf.rules;
import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.math3.random.RandomDataGenerator;
import org.apache.phoenix.pherf.PherfConstants;
import org.apache.commons.lang.RandomStringUtils;
@@ -97,13 +98,6 @@ public class RulesApplier {
Column columnRule = getColumnForRule(ruleList, phxMetaColumn);
value = getDataValue(columnRule);
- synchronized (value) {
- // Add the prefix to the value if it exists.
- if (columnRule.getPrefix() != null) {
- value.setValue(columnRule.getPrefix() + value.getValue());
- }
- }
-
} else {
logger.warn("Attempted to apply rule to data, but could not find a rule to match type:"
+ phxMetaColumn.getType()
@@ -122,46 +116,55 @@ public class RulesApplier {
*/
public DataValue getDataValue(Column column) throws Exception{
DataValue data = null;
+ String prefix = "";
int length = column.getLength();
int nullChance = column.getNullChance();
List<DataValue> dataValues = column.getDataValues();
- // Return an empty value if we we fall within the configured probability
+ // Return an empty value if we fall within the configured probability of null
if ((nullChance != Integer.MIN_VALUE) && (isValueNull(nullChance))) {
return new DataValue(column.getType(), "");
}
+ if (column.getPrefix() != null) {
+ prefix = column.getPrefix();
+ }
+
+ if (prefix.length() >= length) {
+ logger.warn("You are attempting to generate data with a prefix (" + prefix + ") "
+ + "That is longer than expected overall field length (" + length + "). "
+ + "This will certainly lead to unexpected data values.");
+ }
+
switch (column.getType()) {
case VARCHAR:
// Use the specified data values from configs if they exist
if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
- data = generateDataValue(dataValues);
+ data = pickDataValueFromList(dataValues);
} else {
Preconditions.checkArgument(length > 0, "length needs to be > 0");
if (column.getDataSequence() == DataSequence.SEQUENTIAL) {
data = getSequentialDataValue(column);
} else {
- String varchar = RandomStringUtils.randomAlphanumeric(length);
- data = new DataValue(column.getType(), varchar);
+ data = getRandomDataValue(column);
}
}
break;
case CHAR:
if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
- data = generateDataValue(dataValues);
+ data = pickDataValueFromList(dataValues);
} else {
Preconditions.checkArgument(length > 0, "length needs to be > 0");
if (column.getDataSequence() == DataSequence.SEQUENTIAL) {
data = getSequentialDataValue(column);
} else {
- String varchar = RandomStringUtils.randomAlphanumeric(length);
- data = new DataValue(column.getType(), varchar);
+ data = getRandomDataValue(column);
}
}
break;
case DECIMAL:
if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
- data = generateDataValue(dataValues);
+ data = pickDataValueFromList(dataValues);
} else {
int precision = column.getPrecision();
double minDbl = column.getMinValue();
@@ -181,7 +184,7 @@ public class RulesApplier {
break;
case INTEGER:
if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
- data = generateDataValue(dataValues);
+ data = pickDataValueFromList(dataValues);
} else {
int minInt = column.getMinValue();
int maxInt = column.getMaxValue();
@@ -192,7 +195,7 @@ public class RulesApplier {
break;
case DATE:
if ((column.getDataValues() != null) && (column.getDataValues().size() > 0)) {
- data = generateDataValue(dataValues);
+ data = pickDataValueFromList(dataValues);
} else {
int minYear = column.getMinValue();
int maxYear = column.getMaxValue();
@@ -205,7 +208,8 @@ public class RulesApplier {
default:
break;
}
- Preconditions.checkArgument(data != null, "Data value could not be generated for some reason. Please check configs");
+ Preconditions.checkArgument(data != null,
+ "Data value could not be generated for some reason. Please check configs");
return data;
}
@@ -246,7 +250,7 @@ public class RulesApplier {
return (rndNull.nextInt(100) < chance);
}
- private DataValue generateDataValue(List<DataValue> values) throws Exception{
+ private DataValue pickDataValueFromList(List<DataValue> values) throws Exception{
DataValue generatedDataValue = null;
int sum = 0, count = 0;
@@ -265,7 +269,7 @@ public class RulesApplier {
int rndIndex = rndVal.nextInt(values.size());
DataValue valueRule = values.get(rndIndex);
- generatedDataValue = generateDataValue(valueRule);
+ generatedDataValue = pickDataValueFromList(valueRule);
// While it's possible to get here if you have a bunch of really small distributions,
// It's just really unlikely. This is just a safety just so we actually pick a value.
@@ -278,7 +282,7 @@ public class RulesApplier {
return generatedDataValue;
}
- private DataValue generateDataValue(final DataValue valueRule) throws Exception{
+ private DataValue pickDataValueFromList(final DataValue valueRule) throws Exception{
DataValue retValue = new DataValue(valueRule);
// Path taken when configuration specifies a specific value to be taken with the <value> tag
@@ -339,6 +343,14 @@ public class RulesApplier {
}
}
+ public Column getRule(Column phxMetaColumn) {
+ // Assume the first rule map
+ Map<DataTypeMapping, List> ruleMap = modelList.get(0);
+
+ List<Column> ruleList = ruleMap.get(phxMetaColumn.getType());
+ return getColumnForRule(ruleList, phxMetaColumn);
+ }
+
private Column getColumnForRule(List<Column> ruleList, Column phxMetaColumn) {
// Column pointer to head of list
@@ -371,7 +383,21 @@ public class RulesApplier {
long inc = COUNTER.getAndIncrement();
String strInc = String.valueOf(inc);
String varchar = RandomStringUtils.randomAlphanumeric(column.getLength() - strInc.length());
- data = new DataValue(column.getType(), strInc + varchar);
+ varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc + varchar :
+ strInc + varchar;
+
+ // Truncate string back down if it exceeds length
+ varchar = StringUtils.left(varchar,column.getLength());
+ data = new DataValue(column.getType(), varchar);
return data;
}
+
+ private DataValue getRandomDataValue(Column column) {
+ String varchar = RandomStringUtils.randomAlphanumeric(column.getLength());
+ varchar = (column.getPrefix() != null) ? column.getPrefix() + varchar : varchar;
+
+ // Truncate string back down if it exceeds length
+ varchar = StringUtils.left(varchar, column.getLength());
+ return new DataValue(column.getType(), varchar);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 1f610f6..064c604 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -33,7 +33,6 @@ import java.util.Properties;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
-// TODO This class needs to be cleanup up a bit. I just wanted to get an initial placeholder in.
public class PhoenixUtil {
private static final Logger logger = LoggerFactory.getLogger(PhoenixUtil.class);
private static String zookeeper;
@@ -111,7 +110,9 @@ public class PhoenixUtil {
result = preparedStatement.execute();
connection.commit();
} finally {
- preparedStatement.close();
+ if(preparedStatement != null) {
+ preparedStatement.close();
+ }
}
return result;
}
@@ -125,9 +126,15 @@ public class PhoenixUtil {
connection.commit();
} catch (SQLException e) {
e.printStackTrace();
+ if(preparedStatement != null) {
+ logger.error("Failed to apply schema. Statement (" + preparedStatement.toString() + ")",
+ e.getMessage());
+ }
} finally {
try {
- preparedStatement.close();
+ if (preparedStatement != null) {
+ preparedStatement.close();
+ }
} catch (SQLException e) {
e.printStackTrace();
}
@@ -199,7 +206,7 @@ public class PhoenixUtil {
public synchronized List<Column> getColumnsFromPhoenix(String schemaName, String tableName,
Connection connection) throws SQLException {
- List<Column> columnList = new ArrayList<Column>();
+ List<Column> columnList = new ArrayList<>();
ResultSet resultSet = null;
try {
resultSet = getColumnsMetaData(schemaName, tableName, connection);
@@ -293,14 +300,6 @@ public class PhoenixUtil {
executeStatement("UPDATE STATISTICS " + tableName, scenario);
}
- public MonitorManager loadCustomMonitors(MonitorManager manager) throws Exception {
- Properties
- properties =
- PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
-
- return manager;
- }
-
/**
* Get explain plan for a query
*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
index 936eedb..37bfd47 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
@@ -18,17 +18,13 @@
package org.apache.phoenix.pherf;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.lang3.StringUtils;
import org.apache.phoenix.pherf.configuration.Column;
import org.apache.phoenix.pherf.configuration.DataModel;
import org.apache.phoenix.pherf.configuration.DataSequence;
@@ -43,6 +39,8 @@ import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.junit.Test;
+import static org.junit.Assert.*;
+
public class RuleGeneratorTest {
private static final String matcherScenario = PherfConstants.SCENARIO_ROOT_PATTERN + ".xml";
@@ -179,7 +177,8 @@ public class RuleGeneratorTest {
}
}
- assertTrue("Expected count in increments did not match expected", testSet.size() == (threadCount * increments));
+ assertTrue("Expected count in increments did not match expected",
+ testSet.size() == (threadCount * increments));
}
@Test
@@ -205,6 +204,46 @@ public class RuleGeneratorTest {
}
}
+ @Test
+ public void testRuleOverrides() throws Exception {
+ XMLConfigParser parser = new XMLConfigParser(".*test_scenario.xml");
+ WriteWorkload loader = new WriteWorkload(parser);
+ RulesApplier rulesApplier = loader.getRulesApplier();
+ Scenario scenario = parser.getScenarios().get(0);
+
+ // We should be able to find the correct rule based only on Type and Name combination
+ // Test CHAR
+ Column simPhxCol = new Column();
+ simPhxCol.setName("OTHER_ID");
+ simPhxCol.setType(DataTypeMapping.CHAR);
+
+ // Get the rule we expect to match
+ Column rule = rulesApplier.getRule(simPhxCol);
+ assertEquals("Did not find the correct rule.", rule.getName(), simPhxCol.getName());
+ assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType());
+ assertEquals("Rule contains incorrect length.", rule.getLength(), 8);
+ assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "z0Oxx00");
+
+ DataValue value = rulesApplier.getDataForRule(scenario, simPhxCol);
+ assertEquals("Value returned does not match rule.", value.getValue().length(), 8);
+
+ // Test VARCHAR with RANDOM and prefix
+ simPhxCol.setName("OLDVAL_STRING");
+ simPhxCol.setType(DataTypeMapping.VARCHAR);
+
+ // Get the rule we expect to match
+ rule = rulesApplier.getRule(simPhxCol);
+ assertEquals("Did not find the correct rule.", rule.getName(), simPhxCol.getName());
+ assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType());
+ assertEquals("Rule contains incorrect length.", rule.getLength(), 10);
+ assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "MYPRFX");
+
+ value = rulesApplier.getDataForRule(scenario, simPhxCol);
+ assertEquals("Value returned does not match rule.", value.getValue().length(), 10);
+ assertTrue("Value returned start with prefix.",
+ StringUtils.startsWith(value.getValue(), rule.getPrefix()));
+ }
+
/**
* Asserts that the value field is between the min/max value fields
*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
index 4e6b9d4..2cf8e9b 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
@@ -19,6 +19,7 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
TENANT_ID CHAR(15) NOT NULL,
PARENT_ID CHAR(15) NOT NULL,
CREATED_DATE DATE NOT NULL,
+ OTHER_ID CHAR(15),
FIELD VARCHAR,
OLDVAL_STRING VARCHAR,
NEWVAL_STRING VARCHAR,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
index f23e71a..def55ca 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
@@ -112,7 +112,7 @@
<column>
<!-- This column type defines what will generally happen to VARCHAR fields unless they are explicitly defined or overridden elsewhere -->
<type>VARCHAR</type>
- <length>15</length>
+ <length>10</length>
<userDefined>true</userDefined>
<dataSequence>RANDOM</dataSequence>
<name>OLDVAL_STRING</name>
@@ -123,7 +123,7 @@
<type>VARCHAR</type>
<length>15</length>
<userDefined>true</userDefined>
- <dataSequence>SEQUENTIAL</dataSequence>
+ <dataSequence>RANDOM</dataSequence>
<name>NEWVAL_STRING</name>
<prefix>TSTPRFX</prefix>
</column>
@@ -145,7 +145,15 @@
<value>LMN</value>
</datavalue>
</valuelist>
- </column>
+ </column>
+ <column>
+ <type>CHAR</type>
+ <userDefined>true</userDefined>
+ <dataSequence>SEQUENTIAL</dataSequence>
+ <length>8</length>
+ <name>OTHER_ID</name>
+ <prefix>z0Oxx00</prefix>
+ </column>
</datamapping>
<scenarios>
<scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW">