You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/03 07:52:37 UTC
svn commit: r1478649 - in /hive/branches/HIVE-4115/ql/src:
java/org/apache/hadoop/hive/ql/cube/parse/
test/org/apache/hadoop/hive/ql/cube/parse/
test/org/apache/hadoop/hive/ql/cube/processors/
Author: amareshwari
Date: Fri May 3 05:52:37 2013
New Revision: 1478649
URL: http://svn.apache.org/r1478649
Log:
Add least partition resolver
Modified:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java Fri May 3 05:52:37 2013
@@ -66,8 +66,6 @@ public class CubeQueryContext {
private final Map<String, List<String>> tblAliasToColumns = new HashMap<String, List<String>>();
private final Set<String> cubeColumnsQueried = new HashSet<String>();
private final Map<String, String> columnToTabAlias = new HashMap<String, String>();
- protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap =
- new HashMap<CubeFactTable, Map<UpdatePeriod, List<String>>>();
private final Map<CubeQueryExpr, Set<String>> exprToCols = new HashMap<CubeQueryExpr, Set<String>>();
private final Map<CubeQueryExpr, Set<String>> queryExprToExprs = new HashMap<CubeQueryExpr, Set<String>>();
private final Map<String, String> exprToAlias = new HashMap<String, String>();
@@ -75,6 +73,8 @@ public class CubeQueryContext {
private final Set<String> aggregateExprs = new HashSet<String>();
// storage specific
+ protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap =
+ new HashMap<CubeFactTable, Map<UpdatePeriod, List<String>>>();
private List<String> supportedStorages;
private boolean allStoragesSupported;
private final Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factStorageMap =
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java Fri May 3 05:52:37 2013
@@ -1,19 +1,54 @@
package org.apache.hadoop.hive.ql.cube.parse;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
+import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
import org.apache.hadoop.hive.ql.parse.SemanticException;
public class LeastPartitionResolver implements ContextRewriter {
public LeastPartitionResolver(Configuration conf) {
- // TODO Auto-generated constructor stub
}
@Override
public void rewriteContext(CubeQueryContext cubeql)
throws SemanticException {
- // TODO Auto-generated method stub
-
+ if (cubeql.getCube() != null) {
+ Map<CubeFactTable, Integer> numPartitionsMap =
+ new HashMap<CubeFactTable, Integer>();
+
+ for (CubeFactTable fact : cubeql.getCandidateFactTables()) {
+ numPartitionsMap.put(fact, getTotalPartitions(
+ cubeql.getFactPartitionMap().get(fact)));
+ }
+
+ int minPartitions = Collections.min(numPartitionsMap.values());
+
+ for (Iterator<CubeFactTable> i =
+ cubeql.getCandidateFactTables().iterator(); i.hasNext();) {
+ CubeFactTable fact = i.next();
+ if (numPartitionsMap.get(fact) > minPartitions) {
+ System.out.println("Removing fact:" + fact +
+ " from candidate fact tables as it requires more partitions to" +
+ " be queried:" + numPartitionsMap.get(fact) + " minimum:"
+ + minPartitions);
+ i.remove();
+ }
+ }
+ }
}
+ private int getTotalPartitions(Map<UpdatePeriod, List<String>> map) {
+ int count = 0;
+ for (List<String> parts : map.values()) {
+ count += parts.size();
+ }
+ return count;
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java Fri May 3 05:52:37 2013
@@ -112,6 +112,32 @@ public class CubeTestSetup {
storageAggregatePeriods);
}
+ private void createCubeFactOnlyHourly(CubeMetastoreClient client) throws HiveException {
+ String factName = "testFact2";
+ List<FieldSchema> factColumns = new ArrayList<FieldSchema>(
+ cubeMeasures.size());
+ for (CubeMeasure measure : cubeMeasures) {
+ factColumns.add(measure.getColumn());
+ }
+
+ // add dimensions of the cube
+ factColumns.add(new FieldSchema("zipcode","int", "zip"));
+ factColumns.add(new FieldSchema("cityid","int", "city id"));
+
+ Map<Storage, List<UpdatePeriod>> storageAggregatePeriods =
+ new HashMap<Storage, List<UpdatePeriod>>();
+ List<UpdatePeriod> updates = new ArrayList<UpdatePeriod>();
+ updates.add(UpdatePeriod.HOURLY);
+ Storage hdfsStorage = new HDFSStorage("C1",
+ TextInputFormat.class.getCanonicalName(),
+ HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ storageAggregatePeriods.put(hdfsStorage, updates);
+
+ // create cube fact
+ client.createCubeFactTable(cubeName, factName, factColumns,
+ storageAggregatePeriods);
+ }
+
private void createCubeFactMonthly(CubeMetastoreClient client) throws HiveException {
String factName = "testFactMonthly";
List<FieldSchema> factColumns = new ArrayList<FieldSchema>(
@@ -169,6 +195,7 @@ public class CubeTestSetup {
new HiveConf(this.getClass()));
createCube(client);
createCubeFact(client);
+ createCubeFactOnlyHourly(client);
createDimWithTwoStorages(client);
createCubeFactMonthly(client);
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1478649&r1=1478648&r2=1478649&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java Fri May 3 05:52:37 2013
@@ -89,7 +89,9 @@ public class TestCubeDriver {
conf = new Configuration();
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
System.out.println("Test from:" + getDateUptoHours(twodaysBack) + " to:" + getDateUptoHours(now));
- //String expected = "select SUM(testCube.msr2) from "
+ //String expected = " sum( testcube.msr2 ) FROM C1_testfact_HOURLY testcube WHERE " + whereClause(HOURLY) + " UNION " +
+ // SELECT sum( testcube.msr2 ) FROM C1_testfact_DAILY testcube WHERE + whereClause(DAILY)
+
String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
" where time_range_in('" + getDateUptoHours(twodaysBack)
+ "','" + getDateUptoHours(now) + "')");