You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2015/08/23 19:15:52 UTC
[27/40] phoenix git commit: PHOENIX-1791. Adding ability for Pherf
Write Workloads to write to a multi-tenant view.
PHOENIX-1791. Adding ability for Pherf Write Workloads to write to a multi-tenant view.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c811cf68
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c811cf68
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c811cf68
Branch: refs/heads/calcite
Commit: c811cf68752b91585fc9c12741f3be005fefbe2f
Parents: 60500f2
Author: Jan <jf...@salesforce.com>
Authored: Thu Aug 6 11:12:52 2015 -0700
Committer: Cody Marcel <co...@gmail.com>
Committed: Mon Aug 10 10:02:18 2015 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/pherf/DataIngestIT.java | 67 +++++++++++++++++---
.../phoenix/pherf/configuration/Scenario.java | 25 ++++++--
.../pherf/configuration/XMLConfigParser.java | 23 ++++---
.../apache/phoenix/pherf/util/PhoenixUtil.java | 30 +++++----
.../phoenix/pherf/workload/WriteWorkload.java | 49 +++++++++-----
.../datamodel/test_schema_mt_table.sql | 31 +++++++++
.../resources/datamodel/test_schema_mt_view.sql | 1 +
.../test/resources/scenario/test_scenario.xml | 22 +++++++
8 files changed, 198 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
index 1defbb1..297f882 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
@@ -22,6 +22,19 @@ import com.jcabi.jdbc.JdbcSession;
import com.jcabi.jdbc.Outcome;
import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
import org.apache.phoenix.pherf.configuration.Column;
import org.apache.phoenix.pherf.configuration.DataModel;
import org.apache.phoenix.pherf.configuration.DataTypeMapping;
@@ -35,15 +48,8 @@ import org.apache.phoenix.pherf.workload.WriteWorkload;
import org.junit.Before;
import org.junit.Test;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
+import com.jcabi.jdbc.JdbcSession;
+import com.jcabi.jdbc.Outcome;
public class DataIngestIT extends ResultBaseTestIT {
@@ -146,4 +152,47 @@ public class DataIngestIT extends ResultBaseTestIT {
fail("Failed to load data. An exception was thrown: " + e.getMessage());
}
}
+
+
+ @Test
+ /**
+ * Validates that Pherf can write data to a Multi-Tenant View in addition to
+ * standard Phoenix tables.
+ */
+ public void testMultiTenantViewWriteWorkload() throws Exception {
+ // Arrange
+ Scenario scenario = parser.getScenarioByName("testMTWriteScenario");
+ WorkloadExecutor executor = new WorkloadExecutor();
+ executor.add(new WriteWorkload(util, parser, scenario, GeneratePhoenixStats.NO));
+
+ // Act
+ try {
+ // Wait for data to load up.
+ executor.get();
+ executor.shutdown();
+ } catch (Exception e) {
+ fail("Failed to load data. An exception was thrown: " + e.getMessage());
+ }
+
+ assertExpectedNumberOfRecordsWritten(scenario);
+ }
+
+ private void assertExpectedNumberOfRecordsWritten(Scenario scenario) throws Exception,
+ SQLException {
+ Connection connection = util.getConnection(scenario.getTenantId());
+ String sql = "select count(*) from " + scenario.getTableName();
+ Integer count = new JdbcSession(connection).sql(sql).select(new Outcome<Integer>() {
+ @Override public Integer handle(ResultSet resultSet, Statement statement)
+ throws SQLException {
+ while (resultSet.next()) {
+ return resultSet.getInt(1);
+ }
+ return null;
+ }
+ });
+ assertNotNull("Could not retrieve count. " + count);
+ assertEquals("Expected 100 rows to have been inserted",
+ scenario.getRowCount(), count.intValue());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
index 7de96cc..6c949d8 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
@@ -18,15 +18,16 @@
package org.apache.phoenix.pherf.configuration;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.phoenix.pherf.util.PhoenixUtil;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
@XmlRootElement(namespace = "org.apache.phoenix.pherf.configuration.DataModel")
public class Scenario {
@@ -37,6 +38,7 @@ public class Scenario {
private List<QuerySet> querySet = new ArrayList<>();
private WriteParams writeParams;
private String name;
+ private String tenantId;
public Scenario() {
writeParams = new WriteParams();
@@ -162,6 +164,19 @@ public class Scenario {
public void setName(String name) {
this.name = name;
}
+
+ /**
+ * Tenant Id used by connection of this query
+ * @return
+ */
+ @XmlAttribute
+ public String getTenantId() {
+ return tenantId;
+ }
+
+ public void setTenantId(String tenantId) {
+ this.tenantId = tenantId;
+ }
public WriteParams getWriteParams() {
return writeParams;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
index 393fa7e..93dc94c 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
@@ -18,16 +18,6 @@
package org.apache.phoenix.pherf.configuration;
-import org.apache.phoenix.pherf.PherfConstants;
-import org.apache.phoenix.pherf.exception.FileLoaderException;
-import org.apache.phoenix.pherf.util.ResourceList;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -35,6 +25,17 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.exception.FileLoaderException;
+import org.apache.phoenix.pherf.util.ResourceList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class XMLConfigParser {
private static final Logger logger = LoggerFactory.getLogger(XMLConfigParser.class);
@@ -134,6 +135,8 @@ public class XMLConfigParser {
if (fullTableName.contains(".")) {
ret = fullTableName.substring(fullTableName.indexOf(".") + 1, fullTableName.length());
}
+ // Remove any quotes that may be needed for multi-tenant tables
+ ret = ret.replaceAll("\"", "");
return ret;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 5b223b1..19b6bd2 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -18,21 +18,29 @@
package org.apache.phoenix.pherf.util;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
import org.apache.phoenix.pherf.PherfConstants;
import org.apache.phoenix.pherf.configuration.Column;
import org.apache.phoenix.pherf.configuration.DataTypeMapping;
-
-import java.sql.*;
-import java.util.*;
-
import org.apache.phoenix.pherf.configuration.Query;
import org.apache.phoenix.pherf.configuration.QuerySet;
+import org.apache.phoenix.pherf.configuration.Scenario;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
-
// TODO This class needs to be cleanup up a bit. I just wanted to get an initial placeholder in.
public class PhoenixUtil {
private static final Logger logger = LoggerFactory.getLogger(PhoenixUtil.class);
@@ -80,11 +88,11 @@ public class PhoenixUtil {
return DriverManager.getConnection(url, props);
}
- public boolean executeStatement(String sql) throws Exception {
+ public boolean executeStatement(String sql, Scenario scenario) throws Exception {
Connection connection = null;
boolean result = false;
try {
- connection = getConnection();
+ connection = getConnection(scenario.getTenantId());
result = executeStatement(sql, connection);
} finally {
if (connection != null) {
@@ -262,8 +270,8 @@ public class PhoenixUtil {
* @param tableName
* @throws Exception
*/
- public void updatePhoenixStats(String tableName) throws Exception {
+ public void updatePhoenixStats(String tableName, Scenario scenario) throws Exception {
logger.info("Updating stats for " + tableName);
- executeStatement("UPDATE STATISTICS " + tableName);
+ executeStatement("UPDATE STATISTICS " + tableName, scenario);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index b6686c6..d0b99af 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -18,6 +18,21 @@
package org.apache.phoenix.pherf.workload;
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
import org.apache.phoenix.pherf.PherfConstants;
import org.apache.phoenix.pherf.PherfConstants.GeneratePhoenixStats;
import org.apache.phoenix.pherf.configuration.Column;
@@ -35,17 +50,6 @@ import org.apache.phoenix.pherf.util.RowCalculator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.math.BigDecimal;
-import java.sql.*;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
public class WriteWorkload implements Workload {
private static final Logger logger = LoggerFactory.getLogger(WriteWorkload.class);
private final PhoenixUtil pUtil;
@@ -163,15 +167,18 @@ public class WriteWorkload implements Workload {
List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario);
waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches);
-
+
// Update Phoenix Statistics
if (this.generateStatistics == GeneratePhoenixStats.YES) {
logger.info("Updating Phoenix table statistics...");
- pUtil.updatePhoenixStats(scenario.getTableName());
+ pUtil.updatePhoenixStats(scenario.getTableName(), scenario);
logger.info("Stats update done!");
} else {
logger.info("Phoenix table stats update not requested.");
}
+
+ // always update stats for Phoenix base tables
+ updatePhoenixStats(scenario.getTableName(), scenario);
}
private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario)
@@ -185,7 +192,7 @@ public class WriteWorkload implements Workload {
List<Column>
phxMetaCols =
pUtil.getColumnsFromPhoenix(scenario.getSchemaName(),
- scenario.getTableNameWithoutSchemaName(), pUtil.getConnection());
+ scenario.getTableNameWithoutSchemaName(), pUtil.getConnection(scenario.getTenantId()));
int threadRowCount = rowCalculator.getNext();
logger.info(
"Kick off thread (#" + i + ")for upsert with (" + threadRowCount + ") rows.");
@@ -221,6 +228,18 @@ public class WriteWorkload implements Workload {
.add(scenario.getTableName(), sumRows, (int) (System.currentTimeMillis() - start));
}
+ /**
+ * TODO Move this method to PhoenixUtil
+ * Update Phoenix table stats
+ *
+ * @param tableName
+ * @throws Exception
+ */
+ public void updatePhoenixStats(String tableName, Scenario scenario) throws Exception {
+ logger.info("Updating stats for " + tableName);
+ pUtil.executeStatement("UPDATE STATISTICS " + tableName, scenario);
+ }
+
public Future<Info> upsertData(final Scenario scenario, final List<Column> columns,
final String tableName, final int rowCount,
final DataLoadThreadTime dataLoadThreadTime) {
@@ -231,7 +250,7 @@ public class WriteWorkload implements Workload {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Connection connection = null;
try {
- connection = pUtil.getConnection();
+ connection = pUtil.getConnection(scenario.getTenantId());
long logStartTime = System.currentTimeMillis();
long
maxDuration =
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql
new file mode 100644
index 0000000..b6791bf
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql
@@ -0,0 +1,31 @@
+/*
+ -- Licensed to the Apache Software Foundation (ASF) under one
+ -- or more contributor license agreements. See the NOTICE file
+ -- distributed with this work for additional information
+ -- regarding copyright ownership. The ASF licenses this file
+ -- to you under the Apache License, Version 2.0 (the
+ -- "License"); you may not use this file except in compliance
+ -- with the License. You may obtain a copy of the License at
+ --
+ -- http://www.apache.org/licenses/LICENSE-2.0
+ --
+ -- Unless required by applicable law or agreed to in writing, software
+ -- distributed under the License is distributed on an "AS IS" BASIS,
+ -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ -- See the License for the specific language governing permissions and
+ -- limitations under the License.
+*/
+CREATE TABLE IF NOT EXISTS PHERF.TEST_MULTI_TENANT_TABLE (
+ TENANT_ID CHAR(15) NOT NULL,
+ IDENTIFIER CHAR(3) NOT NULL,
+ ID CHAR(15) NOT NULL,
+ CREATED_DATE DATE,
+ FIELD VARCHAR,
+ SOME_INT INTEGER
+ CONSTRAINT PK PRIMARY KEY
+ (
+ TENANT_ID,
+ IDENTIFIER,
+ ID
+ )
+) VERSIONS=1,MULTI_TENANT=true
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql
new file mode 100644
index 0000000..5f5d7ec
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql
@@ -0,0 +1 @@
+CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM PHERF.TEST_MULTI_TENANT_TABLE
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c811cf68/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
index fddf022..b5fe564 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
@@ -127,6 +127,25 @@
<name>NEWVAL_STRING</name>
<prefix>TSTPRFX</prefix>
</column>
+ <column>
+ <type>CHAR</type>
+ <length>3</length>
+ <userDefined>true</userDefined>
+ <dataSequence>LIST</dataSequence>
+ <name>IDENTIFIER</name>
+ <valuelist>
+ <!-- Distributes according to specified values. These must total 100 -->
+ <datavalue distribution="60">
+ <value>ABC</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>XYZ</value>
+ </datavalue>
+ <datavalue distribution="20">
+ <value>LMN</value>
+ </datavalue>
+ </valuelist>
+ </column>
</datamapping>
<scenarios>
<scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW">
@@ -201,5 +220,8 @@
<query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/>
</querySet>
</scenario>
+ <!-- Test writing to a Multi-tenant View -->
+ <scenario tableName="PHERF.TEST_VIEW" tenantId="abcdefghijklmno" rowCount="100" name="testMTWriteScenario">
+ </scenario>
</scenarios>
</datamodel>
\ No newline at end of file