You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by co...@apache.org on 2015/04/24 00:29:46 UTC
[1/2] phoenix git commit: PHOENIX-1728 Pherf - Make tests use mini
cluster
Repository: phoenix
Updated Branches:
refs/heads/master bb30ed516 -> ee5b6707b
PHOENIX-1728 Pherf - Make tests use mini cluster
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/254024f9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/254024f9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/254024f9
Branch: refs/heads/master
Commit: 254024f9c97ffa541fbc097a89e2bd2859fa02a2
Parents: 136f7ee
Author: cmarcel <cm...@salesforce.com>
Authored: Thu Apr 23 15:26:48 2015 -0700
Committer: cmarcel <cm...@salesforce.com>
Committed: Thu Apr 23 15:26:48 2015 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/pherf/DataIngestIT.java | 88 ++++++++++++++++++++
.../apache/phoenix/pherf/SchemaReaderIT.java | 77 +++++++++++++++++
.../apache/phoenix/pherf/RowCalculatorTest.java | 88 ++++++++++++++++++++
3 files changed, 253 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/254024f9/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
new file mode 100644
index 0000000..b29656d
--- /dev/null
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataTypeMapping;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.loaddata.DataLoader;
+import org.apache.phoenix.pherf.rules.DataValue;
+import org.apache.phoenix.pherf.rules.RulesApplier;
+import org.apache.phoenix.pherf.schema.SchemaReader;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.junit.Test;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class DataIngestIT extends BaseHBaseManagedTimeIT {
+ protected static PhoenixUtil util = new PhoenixUtil(true);
+ static final String matcherScenario = ".*scenario/.*test.*xml";
+ static final String matcherSchema = ".*datamodel/.*test.*sql";
+
+ @Test
+ public void generateData() throws Exception {
+ util.setZookeeper("localhost");
+ SchemaReader reader = new SchemaReader(util, matcherSchema);
+ XMLConfigParser parser = new XMLConfigParser(matcherScenario);
+
+ // 1. Generate table schema from file
+ List<Path> resources = new ArrayList<>(reader.getResourceList());
+ assertTrue("Could not pull list of schema files.", resources.size() > 0);
+ assertNotNull("Could not read schema file.", reader.resourceToString(resources.get(0)));
+ reader.applySchema();
+
+ // 2. Load the metadata of for the test tables
+ Scenario scenario = parser.getScenarios().get(0);
+ List<Column> columnListFromPhoenix = util.getColumnsFromPhoenix(scenario.getSchemaName(), scenario.getTableNameWithoutSchemaName(), util.getConnection());
+ assertTrue("Could not get phoenix columns.", columnListFromPhoenix.size() > 0);
+ DataLoader loader = new DataLoader(util,parser);
+ RulesApplier rulesApplier = loader.getRulesApplier();
+ List<Map> modelList = rulesApplier.getModelList();
+ assertTrue("Could not generate the modelList", modelList.size() > 0);
+
+ for (Column column : columnListFromPhoenix) {
+ DataValue data = rulesApplier.getDataForRule(scenario, column);
+
+ // We are generating data values so the value should have been specified by this point.
+ assertTrue("Failed to retrieve data for column type: " + column.getType(), data != null);
+
+ // Test that we still retrieve the GENERAL_CHAR rule even after an override is applied to another CHAR type.
+ // NEWVAL_STRING Column does not specify an override so we should get the default rule.
+ if ((column.getType() == DataTypeMapping.VARCHAR) && (column.getName().equals("NEWVAL_STRING"))) {
+ assertTrue("Failed to retrieve data for column type: ", data.getDistribution() == Integer.MIN_VALUE);
+ }
+ }
+
+ // Load up the data.
+ try {
+ loader.execute();
+ } catch (Exception e) {
+ fail("Failed to lead data. An exception was thrown: " + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/254024f9/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java
new file mode 100644
index 0000000..2cb7c13
--- /dev/null
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/SchemaReaderIT.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.pherf.configuration.Column;
+import org.apache.phoenix.pherf.configuration.DataModel;
+import org.apache.phoenix.pherf.configuration.Scenario;
+import org.apache.phoenix.pherf.configuration.XMLConfigParser;
+import org.apache.phoenix.pherf.schema.SchemaReader;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
+import org.junit.Test;
+
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class SchemaReaderIT extends BaseHBaseManagedTimeIT {
+ protected static PhoenixUtil util = new PhoenixUtil(true);
+
+ @Test
+ public void testSchemaReader() {
+ // Test for the unit test version of the schema files.
+ assertApplySchemaTest();
+ }
+
+ private void assertApplySchemaTest() {
+ try {
+ util.setZookeeper("localhost");
+ SchemaReader reader = new SchemaReader(util, ".*datamodel/.*test.*sql");
+
+ List<Path> resources = new ArrayList<>(reader.getResourceList());
+ assertTrue("Could not pull list of schema files.", resources.size() > 0);
+ assertNotNull("Could not read schema file.", this.getClass().getResourceAsStream(
+ PherfConstants.RESOURCE_DATAMODEL + "/" + resources.get(0).getFileName().toString()));
+ assertNotNull("Could not read schema file.", reader.resourceToString(resources.get(0)));
+ reader.applySchema();
+
+ Connection connection = null;
+ URL resourceUrl = getClass().getResource("/scenario/test_scenario.xml");
+ assertNotNull("Test data XML file is missing", resourceUrl);
+ connection = util.getConnection();
+ Path resourcePath = Paths.get(resourceUrl.toURI());
+ DataModel data = XMLConfigParser.readDataModel(resourcePath);
+ List<Scenario> scenarioList = data.getScenarios();
+ Scenario scenario = scenarioList.get(0);
+ List<Column> columnList = util.getColumnsFromPhoenix(scenario.getSchemaName(), scenario.getTableNameWithoutSchemaName(), connection);
+ assertTrue("Could not retrieve Metadata from Phoenix", columnList.size() > 0);
+ } catch (Exception e) {
+ fail("Could not initialize SchemaReader");
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/254024f9/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java
new file mode 100644
index 0000000..8840289
--- /dev/null
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RowCalculatorTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.pherf;
+
+import org.apache.phoenix.pherf.util.RowCalculator;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+
+public class RowCalculatorTest {
+ /**
+ * Test rows divide evenly with large rows and small threadpool
+ * @throws Exception
+ */
+ @Test
+ public void testRowsEvenDivide() throws Exception {
+ int threadPoolSize = 10;
+ int tableRowCount = 100;
+ assertRowsSum(threadPoolSize, tableRowCount);
+ }
+
+ /**
+ * Test rows add up when not divided evenly with large rows and small threadpool
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testRowsNotEvenDivide() throws Exception {
+ int threadPoolSize = 9;
+ int tableRowCount = 100;
+ assertRowsSum(threadPoolSize, tableRowCount);
+ }
+
+ /**
+ * Test rows add up when not divided evenly with large threadpool and small rowcount
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testRowsNotEvenDivideSmallRC() throws Exception {
+ int threadPoolSize = 50;
+ int tableRowCount = 21;
+ assertRowsSum(threadPoolSize, tableRowCount);
+ }
+
+ /**
+ * Test rows count equal to thread pool
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testRowsEqualToPool() throws Exception {
+ int threadPoolSize = 50;
+ int tableRowCount = 50;
+ assertRowsSum(threadPoolSize, tableRowCount);
+ }
+
+ private void assertRowsSum(int threadPoolSize, int tableRowCount) {
+ int sum = 0;
+ RowCalculator rc = new RowCalculator(threadPoolSize, tableRowCount);
+ assertEquals("Rows generated did not match expected count! ", threadPoolSize, rc.size());
+
+ // Sum of all rows should equal expected row count
+ for (int i = 0; i < threadPoolSize; i++) {
+ sum += rc.getNext();
+ }
+ assertEquals("Rows did not sum up correctly", tableRowCount, sum);
+
+ // Ensure rows were removed from list
+ assertEquals(rc.size(), 0);
+ }
+}
[2/2] phoenix git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/phoenix
Posted by co...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/phoenix
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ee5b6707
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ee5b6707
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ee5b6707
Branch: refs/heads/master
Commit: ee5b6707ba3717f3806584daaf23706ef7c8bd4b
Parents: 254024f bb30ed5
Author: cmarcel <cm...@salesforce.com>
Authored: Thu Apr 23 15:28:46 2015 -0700
Committer: cmarcel <cm...@salesforce.com>
Committed: Thu Apr 23 15:28:46 2015 -0700
----------------------------------------------------------------------
dev/test-patch.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------