You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/06/05 12:16:30 UTC
svn commit: r1489792 - in /hive/branches/HIVE-4115/ql/src:
java/org/apache/hadoop/hive/ql/cube/metadata/
java/org/apache/hadoop/hive/ql/cube/parse/
java/org/apache/hadoop/hive/ql/cube/processors/
test/org/apache/hadoop/hive/ql/cube/parse/ test/org/apac...
Author: amareshwari
Date: Wed Jun 5 10:16:29 2013
New Revision: 1489792
URL: http://svn.apache.org/r1489792
Log:
Add validations for all the tests in TestCubeDriver
Modified:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/AbstractCubeTable.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtil.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/processors/CubeDriver.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/AbstractCubeTable.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/AbstractCubeTable.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/AbstractCubeTable.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/AbstractCubeTable.java Wed Jun 5 10:16:29 2013
@@ -21,14 +21,14 @@ public abstract class AbstractCubeTable
protected AbstractCubeTable(String name, List<FieldSchema> columns,
Map<String, String> props, double weight) {
- this.name = name;
+ this.name = name.toLowerCase();
this.columns = columns;
this.weight = weight;
this.properties.putAll(props);
}
protected AbstractCubeTable(Table hiveTable) {
- this.name = hiveTable.getTableName();
+ this.name = hiveTable.getTableName().toLowerCase();
this.columns = hiveTable.getCols();
this.properties.putAll(hiveTable.getParameters());
this.weight = getWeight(getProperties(), getName());
@@ -100,4 +100,13 @@ public abstract class AbstractCubeTable
public String toString() {
return getName();
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((getName() == null) ? 0 :
+ getName().hashCode());
+ return result;
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java Wed Jun 5 10:16:29 2013
@@ -68,8 +68,6 @@ public class CubeQueryContext {
new HashMap<String, AbstractCubeTable>();
private final Map<AbstractCubeTable, List<String>> cubeTabToCols =
new HashMap<AbstractCubeTable, List<String>>();
- protected Set<CubeFactTable> candidateFactTables =
- new HashSet<CubeFactTable>();
// fields queried
private final Map<String, List<String>> tblAliasToColumns =
@@ -79,8 +77,6 @@ public class CubeQueryContext {
new HashMap<String, String>();
private final Map<CubeQueryExpr, Set<String>> exprToCols =
new HashMap<CubeQueryExpr, Set<String>>();
- private final Map<CubeQueryExpr, Set<String>> queryExprToExprs =
- new HashMap<CubeQueryExpr, Set<String>>();
private final Map<String, String> exprToAlias = new HashMap<String, String>();
private final Set<String> aggregateCols = new HashSet<String>();
private final Set<String> aggregateExprs = new HashSet<String>();
@@ -88,7 +84,9 @@ public class CubeQueryContext {
new HashMap<QBJoinTree, String>();
// storage specific
- protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>>
+ protected final Set<CubeFactTable> candidateFactTables =
+ new HashSet<CubeFactTable>();
+ protected final Map<CubeFactTable, Map<UpdatePeriod, List<String>>>
factPartitionMap =
new HashMap<CubeFactTable, Map<UpdatePeriod, List<String>>>();
private final Map<CubeFactTable, Map<UpdatePeriod, List<String>>>
@@ -102,13 +100,11 @@ public class CubeQueryContext {
new HashMap<AbstractCubeTable, String>();
// query trees
- private ASTNode fromTree;
private String whereTree;
private String havingTree;
private String orderByTree;
private String selectTree;
private String groupByTree;
- private ASTNode limitTree;
private final ASTNode joinTree;
private ASTNode havingAST;
private ASTNode selectAST;
@@ -275,8 +271,6 @@ public class CubeQueryContext {
}
private void extractColumns() throws SemanticException {
- // columnAliases = new ArrayList<String>();
-
// Check if its 'select * from...'
ASTNode selTree = qb.getParseInfo().getSelForClause(clauseName);
if (selTree.getChildCount() == 1) {
@@ -357,17 +351,18 @@ public class CubeQueryContext {
&& (parent != null && parent.getToken().getType() != DOT)) {
// Take child ident.totext
ASTNode ident = (ASTNode) node.getChild(0);
+ String column = ident.getText().toLowerCase();
if (tblToCols != null) {
List<String> colList = tblToCols.get(DEFAULT_TABLE);
if (colList == null) {
colList = new ArrayList<String>();
tblToCols.put(DEFAULT_TABLE, colList);
}
- if (!colList.contains(ident.getText())) {
- colList.add(ident.getText());
+ if (!colList.contains(column)) {
+ colList.add(column);
}
}
- columns.add(ident.getText());
+ columns.add(column);
} else if (node.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
@@ -377,8 +372,8 @@ public class CubeQueryContext {
Identifier);
ASTNode colIdent = (ASTNode) node.getChild(1);
- String column = colIdent.getText();
- String table = tabident.getText();
+ String column = colIdent.getText().toLowerCase();
+ String table = tabident.getText().toLowerCase();
if (tblToCols != null) {
List<String> colList = tblToCols.get(table);
if (colList == null) {
@@ -397,7 +392,7 @@ public class CubeQueryContext {
ASTNode alias = HQLParser.findNodeByPath(node, Identifier);
if (alias != null) {
exprToAlias.put(HQLParser.getString(node).trim().toLowerCase(),
- alias.getText());
+ alias.getText().toLowerCase());
}
}
}
@@ -430,8 +425,8 @@ public class CubeQueryContext {
+ " in dimensions '" + prevDim + "' and '"
+ dim.getName() + "'");
}
- columnToTabAlias.put(col.toLowerCase(), dim.getName());
- break;
+ columnToTabAlias.put(col.toLowerCase(), getAliasForTabName(
+ dim.getName()));
} else {
// throw error because column is in both cube and dimension table
throw new SemanticException("Ambiguous column:" + col
@@ -882,8 +877,8 @@ public class CubeQueryContext {
CubeDimensionTable dim = it.next();
appendWhereClause(dim, whereWithoutTimerange, factStorageTable != null);
while (it.hasNext()) {
- appendWhereClause(dim, whereWithoutTimerange, true);
dim = it.next();
+ appendWhereClause(dim, whereWithoutTimerange, true);
}
}
if (whereWithoutTimerange.length() == 0) {
@@ -921,7 +916,7 @@ public class CubeQueryContext {
storageTableToQuery.put(getCube(), storageTable);
query.append(toHQL(storageTable));
if (it.hasNext()) {
- query.append(" UNION ");
+ query.append(" UNION ALL ");
}
}
return query.toString();
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtil.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtil.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtil.java Wed Jun 5 10:16:29 2013
@@ -168,11 +168,6 @@ public class DateUtil {
hasFraction = true;
break;
}
- case WEEKLY:
- if (cal.get(Calendar.DAY_OF_WEEK) != 1) {
- hasFraction = true;
- break;
- }
case DAILY:
if (cal.get(Calendar.HOUR_OF_DAY) != 0) {
hasFraction = true;
@@ -191,6 +186,11 @@ public class DateUtil {
case SECONDLY:
if (cal.get(Calendar.MILLISECOND) != 0) {
hasFraction = true;
+ }
+ break;
+ case WEEKLY:
+ if (cal.get(Calendar.DAY_OF_WEEK) != 1) {
+ hasFraction = true;
break;
}
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java Wed Jun 5 10:16:29 2013
@@ -11,6 +11,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -205,7 +206,7 @@ public class StorageTableResolver implem
Date fromDate, Date toDate)
throws SemanticException {
Map<UpdatePeriod, List<String>> partitionColMap =
- new HashMap<UpdatePeriod, List<String>>();
+ new TreeMap<UpdatePeriod, List<String>>();
Set<UpdatePeriod> updatePeriods = factStorageMap.get(fact).keySet();
try {
if (!getPartitions(fact, fromDate, toDate, partitionColMap,
@@ -265,6 +266,7 @@ public class StorageTableResolver implem
interval, dt)) {
partitions.add(part);
foundPart = true;
+ LOG.info("Adding existing partition" + part);
break;
}
}
@@ -301,7 +303,7 @@ public class StorageTableResolver implem
toDate, partitionColMap, updatePeriods, addNonExistingParts));
}
- private static String getWherePartClause(String tableName,
+ public static String getWherePartClause(String tableName,
List<String> parts) {
if (parts.size() == 0) {
return "";
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/processors/CubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/processors/CubeDriver.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/processors/CubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/processors/CubeDriver.java Wed Jun 5 10:16:29 2013
@@ -68,6 +68,7 @@ public class CubeDriver extends Driver {
if (explain) {
hql = "EXPLAIN " + hql;
}
+ System.out.println("Rewritten query:" + hql);
return hql;
}
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java Wed Jun 5 10:16:29 2013
@@ -41,10 +41,14 @@ public class CubeTestSetup {
public static Date now;
public static Date twodaysBack;
public static Date twoMonthsBack;
+ private static boolean zerothHour;
+ private static boolean firstDayOfMonth;
static {
Calendar cal = Calendar.getInstance();
now = cal.getTime();
+ zerothHour = (cal.get(Calendar.HOUR_OF_DAY) == 0);
+ firstDayOfMonth = (cal.get(Calendar.DAY_OF_MONTH) == 0);
System.out.println("Test now:" + now);
cal.add(Calendar.DAY_OF_MONTH, -2);
twodaysBack = cal.getTime();
@@ -55,6 +59,10 @@ public class CubeTestSetup {
System.out.println("Test twoMonthsBack:" + twoMonthsBack);
}
+ public static boolean isZerothHour() {
+ return zerothHour;
+ }
+
private void createCube(CubeMetastoreClient client) throws HiveException {
cubeMeasures = new HashSet<CubeMeasure>();
cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr1", "int",
@@ -199,7 +207,6 @@ public class CubeTestSetup {
cal.setTime(twodaysBack);
Date temp = cal.getTime();
while (!(temp.after(now))) {
- System.out.println("Adding partition" + temp);
client.addPartition(fact2, hdfsStorage,
UpdatePeriod.HOURLY, temp);
cal.add(Calendar.HOUR_OF_DAY, 1);
@@ -425,4 +432,8 @@ public class CubeTestSetup {
client.createCubeTable(fact3, storageAggregatePeriods);
}
+ public static boolean isFirstDayOfMonth() {
+ return firstDayOfMonth;
+ }
+
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1489792&r1=1489791&r2=1489792&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java Wed Jun 5 10:16:29 2013
@@ -1,12 +1,25 @@
package org.apache.hadoop.hive.ql.cube.processors;
+import static org.apache.hadoop.hive.ql.cube.parse.CubeTestSetup.now;
+import static org.apache.hadoop.hive.ql.cube.parse.CubeTestSetup.twoMonthsBack;
+import static org.apache.hadoop.hive.ql.cube.parse.CubeTestSetup.twodaysBack;
+
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.cube.metadata.Storage;
+import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
import org.apache.hadoop.hive.ql.cube.parse.CubeQueryConstants;
import org.apache.hadoop.hive.ql.cube.parse.CubeTestSetup;
+import org.apache.hadoop.hive.ql.cube.parse.DateUtil;
+import org.apache.hadoop.hive.ql.cube.parse.StorageTableResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.junit.Assert;
import org.junit.Before;
@@ -17,6 +30,13 @@ public class TestCubeDriver {
private Configuration conf;
private CubeDriver driver;
+ private final String cubeName = "testcube";
+ private final String twoDaysRange = "time_range_in('" + getDateUptoHours(
+ twodaysBack) + "','" + getDateUptoHours(now) + "')";
+ private final String twoMonthsRangeUptoHours = "time_range_in('" +
+ getDateUptoHours(twoMonthsBack) + "','" + getDateUptoHours(now) + "')";
+ private final String twoMonthsRangeUptoMonth = "time_range_in('" +
+ getDateUptoMonth(twoMonthsBack) + "','" + getDateUptoMonth(now) + "')";
@BeforeClass
public static void setup() throws Exception {
@@ -50,8 +70,8 @@ public class TestCubeDriver {
public void testQueryWithNow() throws Exception {
Throwable th = null;
try {
- String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube"
- + " where time_range_in('NOW - 2DAYS', 'NOW')");
+ driver.compileCubeQuery("select SUM(msr2) from testCube where" +
+ " time_range_in('NOW - 2DAYS', 'NOW')");
} catch (SemanticException e) {
th = e;
e.printStackTrace();
@@ -63,70 +83,255 @@ public class TestCubeDriver {
public void testCandidateTables() throws Exception {
Throwable th = null;
try {
- String hqlQuery = driver.compileCubeQuery("select dim12, SUM(msr2)" +
- " from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
+ driver.compileCubeQuery("select dim12, SUM(msr2) from testCube" +
+ " where " + twoDaysRange);
} catch (SemanticException e) {
th = e;
e.printStackTrace();
}
Assert.assertNotNull(th);
+ try {
+ // this query should through exception because invalidMsr is invalid
+ driver.compileCubeQuery("SELECT cityid, invalidMsr from testCube " +
+ " where " + twoDaysRange);
+ Assert.assertTrue("Should not reach here", false);
+ } catch (SemanticException exc) {
+ exc.printStackTrace();
+ Assert.assertNotNull(exc);
+ }
+
+ }
+
+ public String getExpectedQuery(String cubeName, String selExpr,
+ String whereExpr, String postWhereExpr,
+ Map<String, String> storageTableToWhereClause) {
+ StringBuilder expected = new StringBuilder();
+ int numTabs = storageTableToWhereClause.size();
+ int i = 0;
+ for (Map.Entry<String, String> entry : storageTableToWhereClause.entrySet())
+ {
+ String storageTable = entry.getKey();
+ expected.append(selExpr);
+ expected.append(storageTable);
+ expected.append(" ");
+ expected.append(cubeName);
+ expected.append(" WHERE ");
+ expected.append("(");
+ if (whereExpr != null) {
+ expected.append(whereExpr);
+ expected.append(" AND ");
+ }
+ expected.append(entry.getValue());
+ expected.append(")");
+ if (postWhereExpr != null) {
+ expected.append(postWhereExpr);
+ }
+ if (i < numTabs -1) {
+ expected.append(" UNION ALL ");
+ }
+ i++;
+ }
+ return expected.toString();
+ }
+
+ public String getExpectedQuery(String cubeName, String selExpr,
+ String joinExpr, String whereExpr, String postWhereExpr,
+ List<String> joinWhereConds,
+ Map<String, String> storageTableToWhereClause) {
+ StringBuilder expected = new StringBuilder();
+ int numTabs = storageTableToWhereClause.size();
+ int i = 0;
+ for (Map.Entry<String, String> entry : storageTableToWhereClause.entrySet())
+ {
+ String storageTable = entry.getKey();
+ expected.append(selExpr);
+ expected.append(storageTable);
+ expected.append(" ");
+ expected.append(cubeName);
+ expected.append(joinExpr);
+ expected.append(" WHERE ");
+ expected.append("(");
+ if (whereExpr != null) {
+ expected.append(whereExpr);
+ expected.append(" AND ");
+ }
+ expected.append(entry.getValue());
+ if (joinWhereConds != null) {
+ for (String joinEntry : joinWhereConds) {
+ expected.append(" AND ");
+ expected.append(joinEntry);
+ }
+ }
+ expected.append(")");
+ if (postWhereExpr != null) {
+ expected.append(postWhereExpr);
+ }
+ if (i < numTabs -1) {
+ expected.append(" UNION ALL ");
+ }
+ i++;
+ }
+ return expected.toString();
+ }
+
+ private Map<String, String> getWhereForDailyAndHourly2days(String cubeName,
+ String hourlyTable, String dailyTable) {
+ Map<String, String> storageTableToWhereClause =
+ new LinkedHashMap<String, String>();
+ if (!CubeTestSetup.isZerothHour()) {
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.HOURLY, twodaysBack,
+ DateUtil.getCeilDate(twodaysBack, UpdatePeriod.DAILY));
+ addParts(parts, UpdatePeriod.HOURLY,
+ DateUtil.getFloorDate(now, UpdatePeriod.DAILY),
+ DateUtil.getFloorDate(now, UpdatePeriod.HOURLY));
+ storageTableToWhereClause.put(hourlyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ }
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.DAILY, DateUtil.getCeilDate(
+ twodaysBack, UpdatePeriod.DAILY),
+ DateUtil.getFloorDate(now, UpdatePeriod.DAILY));
+ storageTableToWhereClause.put(dailyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ return storageTableToWhereClause;
+ }
+
+ private Map<String, String> getWhereForMonthlyDailyAndHourly2months(
+ String hourlyTable, String dailyTable, String monthlyTable) {
+ Map<String, String> storageTableToWhereClause =
+ new LinkedHashMap<String, String>();
+ if (!CubeTestSetup.isZerothHour()) {
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.HOURLY, twoMonthsBack,
+ DateUtil.getCeilDate(twoMonthsBack, UpdatePeriod.DAILY));
+ addParts(parts, UpdatePeriod.HOURLY,
+ DateUtil.getFloorDate(now, UpdatePeriod.DAILY),
+ DateUtil.getFloorDate(now, UpdatePeriod.HOURLY));
+ storageTableToWhereClause.put(hourlyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ }
+ if (!CubeTestSetup.isFirstDayOfMonth()) {
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.DAILY, DateUtil.getCeilDate(
+ twoMonthsBack, UpdatePeriod.DAILY),
+ DateUtil.getCeilDate(twoMonthsBack, UpdatePeriod.MONTHLY));
+ addParts(parts, UpdatePeriod.DAILY,
+ DateUtil.getFloorDate(now, UpdatePeriod.MONTHLY),
+ DateUtil.getFloorDate(now, UpdatePeriod.DAILY));
+ storageTableToWhereClause.put(dailyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ }
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.MONTHLY, DateUtil.getCeilDate(
+ twoMonthsBack, UpdatePeriod.MONTHLY),
+ DateUtil.getFloorDate(now, UpdatePeriod.MONTHLY));
+ storageTableToWhereClause.put(monthlyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ return storageTableToWhereClause;
+ }
+
+ private Map<String, String> getWhereForMonthly2months(String monthlyTable) {
+ Map<String, String> storageTableToWhereClause =
+ new LinkedHashMap<String, String>();
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.MONTHLY,
+ twoMonthsBack,
+ DateUtil.getFloorDate(now, UpdatePeriod.MONTHLY));
+ storageTableToWhereClause.put(monthlyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ return storageTableToWhereClause;
+ }
+
+ private Map<String, String> getWhereForHourly2days(String hourlyTable) {
+ Map<String, String> storageTableToWhereClause =
+ new LinkedHashMap<String, String>();
+ List<String> parts = new ArrayList<String>();
+ addParts(parts, UpdatePeriod.HOURLY, twodaysBack,
+ DateUtil.getFloorDate(now, UpdatePeriod.HOURLY));
+ storageTableToWhereClause.put(hourlyTable,
+ StorageTableResolver.getWherePartClause(cubeName, parts));
+ return storageTableToWhereClause;
+ }
+
+ private void addParts(List<String> partitions, UpdatePeriod updatePeriod,
+ Date from, Date to) {
+ String fmt = updatePeriod.format();
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(from);
+ Date dt = cal.getTime();
+ while (dt.before(to)) {
+ String part = new SimpleDateFormat(fmt).format(dt);
+ cal.add(updatePeriod.calendarField(), 1);
+ partitions.add(part);
+ dt = cal.getTime();
+ }
}
@Test
public void testCubeExplain() throws Exception {
String hqlQuery = driver.compileCubeQuery("explain select SUM(msr2) from " +
- "testCube where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ "testCube where " + twoDaysRange);
+ String expected = "EXPLAIN " + getExpectedQuery(cubeName,
+ "select sum(testcube.msr2) FROM ", null, null,
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
}
+ private void compareQueries(String expected, String actual) {
+ expected = expected.replaceAll("\\W", "");
+ actual = actual.replaceAll("\\W", "");
+ Assert.assertTrue(expected.equalsIgnoreCase(actual));
+ }
@Test
public void testCubeWhereQuery() throws Exception {
- System.out.println("Test from:" + getDateUptoHours(CubeTestSetup.twodaysBack) + " to:" +
- getDateUptoHours(CubeTestSetup.now));
- //String expected = " sum( testcube.msr2 ) FROM C1_testfact_HOURLY
- //testcube WHERE " + whereClause(HOURLY) + " UNION " +
- // SELECT sum( testcube.msr2 ) FROM C1_testfact_DAILY testcube
- //WHERE + whereClause(DAILY)
-
String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ String expected = getExpectedQuery(cubeName,
+ "select sum(testcube.msr2) FROM ", null, null,
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
// Test with partition existence
conf.setBoolean(CubeQueryConstants.FAIL_QUERY_ON_PARTIAL_DATA, true);
+ driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForHourly2days("c1_testfact2_hourly"));
+ compareQueries(expected, hqlQuery);
conf.setBoolean(CubeQueryConstants.FAIL_QUERY_ON_PARTIAL_DATA, false);
// Tests for valid tables
conf.set(CubeQueryConstants.VALID_FACT_TABLES, "testFact");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C2");
conf.set(CubeQueryConstants.VALID_FACT_TABLES, "testFact");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact_HOURLY",
+ "C2_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C1");
conf.set(CubeQueryConstants.VALID_FACT_TABLES, "testFact2");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForHourly2days("c1_testfact2_hourly"));
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.VALID_FACT_TABLES, "");
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C1");
@@ -134,173 +339,213 @@ public class TestCubeDriver {
"C1_testFact2_HOURLY");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForHourly2days("c1_testfact2_hourly"));
+ compareQueries(expected, hqlQuery);
- conf.set(CubeQueryConstants.VALID_STORAGE_FACT_TABLES,
- "C1_testFact_HOURLY");
+ conf.set(CubeQueryConstants.VALID_STORAGE_FACT_TABLES, "C1_testFact_HOURLY");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForHourly2days("c1_testfact_hourly"));
+ compareQueries(expected, hqlQuery);
- conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES,
- "C2");
+ conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C2");
conf.set(CubeQueryConstants.VALID_FACT_TABLES, "");
- conf.set(CubeQueryConstants.VALID_STORAGE_FACT_TABLES,
- "C2_testFact_HOURLY");
+ conf.set(CubeQueryConstants.VALID_STORAGE_FACT_TABLES, "C2_testFact_HOURLY");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
- //Assert.assertEquals(queries[1], cubeql.toHQL());
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForHourly2days("c2_testfact_hourly"));
+ compareQueries(expected, hqlQuery);
}
@Test
public void testCubeJoinQuery() throws Exception {
- //String expected = "select SUM(testCube.msr2) from "
String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube"
- + " join citytable on testCube.cityid = citytable.id"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ + " join citytable on testCube.cityid = citytable.id"
+ + " where " + twoDaysRange);
+ List<String> joinWhereConds = new ArrayList<String>();
+ joinWhereConds.add(StorageTableResolver.getWherePartClause("citytable",
+ Storage.getPartitionsForLatest()));
+ String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2)" +
+ " FROM ", "INNER JOIN c1_citytable citytable ON" +
+ " testCube.cityid = citytable.id", null, null, joinWhereConds,
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select statetable.name, SUM(msr2) from"
- + " testCube"
- + " join citytable on testCube.cityid = citytable.id"
- + " left outer join statetable on statetable.id = citytable.stateid"
- + " right outer join ziptable on citytable.zipcode = ziptable.code"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ + " testCube"
+ + " join citytable on testCube.cityid = citytable.id"
+ + " left outer join statetable on statetable.id = citytable.stateid"
+ + " right outer join ziptable on citytable.zipcode = ziptable.code"
+ + " where " + twoDaysRange);
+ joinWhereConds = new ArrayList<String>();
+ joinWhereConds.add(StorageTableResolver.getWherePartClause("statetable",
+ Storage.getPartitionsForLatest()));
+ joinWhereConds.add(StorageTableResolver.getWherePartClause("citytable",
+ Storage.getPartitionsForLatest()));
+ joinWhereConds.add(StorageTableResolver.getWherePartClause("ziptable",
+ Storage.getPartitionsForLatest()));
+ expected = getExpectedQuery(cubeName, "select statetable.name," +
+ " sum(testcube.msr2) FROM ", "INNER JOIN c1_citytable citytable ON" +
+ " testCube.cityid = citytable.id LEFT OUTER JOIN c1_statetable statetable"
+ + " ON statetable.id = citytable.stateid RIGHT OUTER JOIN c1_ziptable" +
+ " ziptable ON citytable.zipcode = ziptable.code", null, " group by" +
+ " statetable.name ", joinWhereConds,
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube"
- + " join countrytable on testCube.countryid = countrytable.id"
- + " where time_range_in('" + getDateUptoMonth(CubeTestSetup.twoMonthsBack)
- + "','" + getDateUptoMonth(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ + " join countrytable on testCube.countryid = countrytable.id"
+ + " where " + twoMonthsRangeUptoMonth);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ " INNER JOIN c1_countrytable countrytable ON testCube.countryid = " +
+ " countrytable.id", null, null, null,
+ getWhereForMonthly2months("c2_testfactmonthly_monthly"));
+ compareQueries(expected, hqlQuery);
+
+ try {
+ hqlQuery = driver.compileCubeQuery("select name, SUM(msr2) from testCube"
+ + " join citytable" + " where " + twoDaysRange + " group by name");
+ Assert.assertTrue(false);
+ } catch (SemanticException e) {
+ e.printStackTrace();
+ }
+
}
@Test
public void testCubeGroupbyQuery() throws Exception {
- //String expected = "select SUM(testCube.msr2) from "
String hqlQuery = driver.compileCubeQuery("select name, SUM(msr2) from" +
- " testCube"
- + " join citytable on testCube.cityid = citytable.id"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " testCube join citytable on testCube.cityid = citytable.id where " +
+ twoDaysRange);
+ List<String> joinWhereConds = new ArrayList<String>();
+ joinWhereConds.add(StorageTableResolver.getWherePartClause("citytable",
+ Storage.getPartitionsForLatest()));
+ String expected = getExpectedQuery(cubeName, "select citytable.name," +
+ " sum(testcube.msr2) FROM ", "INNER JOIN c1_citytable citytable ON" +
+ " testCube.cityid = citytable.id", null, " group by citytable.name ",
+ joinWhereConds, getWhereForDailyAndHourly2days(cubeName,
+ "C1_testfact_HOURLY", "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube"
- + " join citytable on testCube.cityid = citytable.id"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')"
- + " group by name");
- System.out.println("cube hql:" + hqlQuery);
+ + " join citytable on testCube.cityid = citytable.id"
+ + " where " + twoDaysRange + " group by name");
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select cityid, SUM(msr2) from testCube"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ + " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select testcube.cityid," +
+ " sum(testcube.msr2) FROM ", null, " group by testcube.cityid ",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select round(cityid), SUM(msr2) from" +
- " testCube"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " testCube where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select round(testcube.cityid)," +
+ " sum(testcube.msr2) FROM ", null, " group by round(testcube.cityid) ",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')"
- + " group by round(zipcode)");
+ + " where " + twoDaysRange + "group by round(zipcode)");
+ expected = getExpectedQuery(cubeName, "select round(testcube.zipcode)," +
+ " sum(testcube.msr2) FROM ", null, " group by round(testcube.zipcode) ",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select round(cityid), SUM(msr2) from" +
- " testCube"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')"
- + " group by zipcode");
- System.out.println("cube hql:" + hqlQuery);
+ " testCube where " + twoDaysRange + " group by zipcode");
+ expected = getExpectedQuery(cubeName, "select testcube.zipcode," +
+ " round(testcube.cityid), sum(testcube.msr2) FROM ", null,
+ " group by testcube.zipcode, round(testcube.cityid)",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select cityid, SUM(msr2) from testCube"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')"
- + " group by round(zipcode)");
- System.out.println("cube hql:" + hqlQuery);
+ + " where " + twoDaysRange + " group by round(zipcode)");
+ expected = getExpectedQuery(cubeName, "select round(testcube.zipcode)," +
+ " testcube.cityid, sum(testcube.msr2) FROM ", null,
+ " group by round(testcube.zipcode), testcube.cityid",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select cityid, msr2 from testCube"
- + " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')"
- + " group by round(zipcode)");
- System.out.println("cube hql:" + hqlQuery);
+ + " where " + twoDaysRange + " group by round(zipcode)");
+ expected = getExpectedQuery(cubeName, "select round(testcube.zipcode)," +
+ " testcube.cityid, sum(testcube.msr2) FROM ", null,
+ " group by round(testcube.zipcode), testcube.cityid",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
}
@Test
public void testCubeQueryWithAilas() throws Exception {
- String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
- hqlQuery = driver.compileCubeQuery("select SUM(msr2) m2 from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
- try {
- hqlQuery = driver.compileCubeQuery("select name, SUM(msr2) from testCube"
- + " join citytable" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')" +
- " group by name");
- System.out.println("cube hql:" + hqlQuery);
- } catch (SemanticException e) {
- e.printStackTrace();
- }
- hqlQuery = driver.compileCubeQuery("select SUM(mycube.msr2) from" +
- " testCube mycube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
- //Assert.assertEquals(queries[1], cubeql.toHQL());
-
- hqlQuery = driver.compileCubeQuery("select SUM(testCube.msr2) from" +
- " testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ String hqlQuery = driver.compileCubeQuery("select SUM(msr2) m2 from" +
+ " testCube where " + twoDaysRange);
+ String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2)" +
+ " m2 FROM ", null, null, getWhereForDailyAndHourly2days(cubeName,
+ "C1_testfact_HOURLY", "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
+
+ hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube mycube" +
+ " where " + twoDaysRange);
+ expected = getExpectedQuery("mycube", "select sum(mycube.msr2) FROM ", null,
+ null, getWhereForDailyAndHourly2days("mycube", "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
+
+ hqlQuery = driver.compileCubeQuery("select SUM(testCube.msr2) from testCube"
+ + " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ",
+ null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select mycube.msr2 m2 from testCube" +
- " mycube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " mycube where " + twoDaysRange);
+ expected = getExpectedQuery("mycube", "select sum(mycube.msr2) m2 FROM ",
+ null, null, getWhereForDailyAndHourly2days("mycube", "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select testCube.msr2 m2 from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) m2 FROM ",
+ null, null, getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ compareQueries(expected, hqlQuery);
}
@Test
public void testCubeWhereQueryForMonth() throws Exception {
- System.out.println("Test from:" + getDateUptoHours(
- CubeTestSetup.twoMonthsBack) + " to:" +
- getDateUptoHours(CubeTestSetup.now));
String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twoMonthsBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
- //Assert.assertEquals(queries[1], cubeql.toHQL());
+ " where " + twoMonthsRangeUptoHours);
+ String expected = getExpectedQuery(cubeName,
+ "select sum(testcube.msr2) FROM ", null, null,
+ getWhereForMonthlyDailyAndHourly2months("C1_testfact_HOURLY",
+ "C1_testfact_DAILY", "c1_testfact_monthly"));
+ compareQueries(expected, hqlQuery);
conf.setBoolean(CubeQueryConstants.FAIL_QUERY_ON_PARTIAL_DATA, true);
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
try {
hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoHours(
- CubeTestSetup.twoMonthsBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
+ " where " + twoMonthsRangeUptoHours);
Assert.assertTrue(false);
} catch (SemanticException e) {
e.printStackTrace();
@@ -308,99 +553,167 @@ public class TestCubeDriver {
conf.setBoolean(CubeQueryConstants.FAIL_QUERY_ON_PARTIAL_DATA, false);
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
- // TODO this should consider only two month partitions. Month weight needs
- // to be fixed.
- hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
- " where time_range_in('" + getDateUptoMonth(CubeTestSetup.twoMonthsBack)
- + "','" + getDateUptoMonth(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ // this should consider only two month partitions.
+ hqlQuery = driver.compileCubeQuery("select cityid, SUM(msr2) from testCube"
+ + " where " + twoMonthsRangeUptoMonth);
+ expected = getExpectedQuery(cubeName, "select testcube,cityid," +
+ " sum(testcube.msr2) FROM ", null, "group by testcube.cityid",
+ getWhereForMonthly2months("c1_testfact_monthly"));
+ compareQueries(expected, hqlQuery);
+ }
+
+ String getExpectedQuery(String dimName, String selExpr, String postWhereExpr,
+ String storageTable, boolean hasPart) {
+ StringBuilder expected = new StringBuilder();
+ expected.append(selExpr);
+ expected.append(storageTable);
+ expected.append(" ");
+ expected.append(dimName);
+ if (hasPart) {
+ expected.append(" WHERE ");
+ expected.append(StorageTableResolver.getWherePartClause(dimName,
+ Storage.getPartitionsForLatest()));
+ }
+ if (postWhereExpr != null) {
+ expected.append(postWhereExpr);
+ }
+ return expected.toString();
}
@Test
public void testDimensionQueryWithMultipleStorages() throws Exception {
String hqlQuery = driver.compileCubeQuery("select name, stateid from" +
- " citytable");
- System.out.println("cube hql:" + hqlQuery);
+ " citytable");
+ String expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", null, "c1_citytable", true);
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select name, c.stateid from citytable" +
- " c");
- System.out.println("cube hql:" + hqlQuery);
+ " c");
+ expected = getExpectedQuery("c", "select c.name, c.stateid from ", null,
+ "c1_citytable", true);
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C2");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select name, stateid from citytable");
- System.out.println("cube hql:" + hqlQuery);
- //Assert.assertEquals(queries[1], cubeql.toHQL());
+ expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", null, "c2_citytable", false);
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C1");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select name, stateid from citytable");
- System.out.println("cube hql:" + hqlQuery);
+ expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", null, "c1_citytable", true);
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "");
conf.set(CubeQueryConstants.VALID_STORAGE_DIM_TABLES, "C1_citytable");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select name, stateid from citytable");
- System.out.println("cube hql:" + hqlQuery);
+ expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", null, "c1_citytable", true);
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "");
conf.set(CubeQueryConstants.VALID_STORAGE_DIM_TABLES, "C2_citytable");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select name, stateid from citytable");
- System.out.println("cube hql:" + hqlQuery);
+ expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", null, "c2_citytable", false);
+ compareQueries(expected, hqlQuery);
}
@Test
public void testLimitQueryOnDimension() throws Exception {
String hqlQuery = driver.compileCubeQuery("select name, stateid from" +
- " citytable limit 100");
- System.out.println("cube hql:" + hqlQuery);
- //Assert.assertEquals(queries[1], cubeql.toHQL());
+ " citytable limit 100");
+ String expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", " limit 100", "c1_citytable", true);
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C2");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select name, stateid from citytable " +
- "limit 100");
- System.out.println("cube hql:" + hqlQuery);
+ "limit 100");
+ expected = getExpectedQuery("citytable", "select citytable.name," +
+ "citytable.stateid from ", " limit 100", "c2_citytable", false);
+ compareQueries(expected, hqlQuery);
conf.set(CubeQueryConstants.DRIVER_SUPPORTED_STORAGES, "C1");
driver = new CubeDriver(new HiveConf(conf, HiveConf.class));
hqlQuery = driver.compileCubeQuery("select name, stateid from citytable" +
- " limit 100");
- System.out.println("cube hql:" + hqlQuery);
+ " limit 100");
+ expected = getExpectedQuery("citytable", "select citytable.name," +
+ " citytable.stateid from ", " limit 100", "c1_citytable", true);
+ compareQueries(expected, hqlQuery);
}
@Test
public void testAggregateResolver() throws Exception {
- String timeRange = " where time_range_in('2013-05-01', '2013-05-03')";
- System.out.println("#$AGGREGATE_RESOLVER_ TIME_RANGE:" + timeRange);
- String q1 = "SELECT cityid, testCube.msr2 from testCube " + timeRange;
- String q2 = "SELECT cityid, testCube.msr2 * testCube.msr2 from testCube "
- + timeRange;
- String q3 = "SELECT cityid, sum(testCube.msr2) from testCube " + timeRange;
- String q4 = "SELECT cityid, sum(testCube.msr2) from testCube " + timeRange
- + " having testCube.msr2 > 100";
- String q5 = "SELECT cityid, testCube.msr2 from testCube " + timeRange
- + " having testCube.msr2 + testCube.msr2 > 100";
- String q6 = "SELECT cityid, testCube.msr2 from testCube " + timeRange
- + " having testCube.msr2 > 100 AND testCube.msr2 < 100";
- String q7 = "SELECT cityid, sum(testCube.msr2) from testCube " + timeRange
- + " having (testCube.msr2 > 100) OR (testcube.msr2 < 100 AND " +
- "SUM(testcube.msr3) > 1000)";
+ String q1 = "SELECT cityid, testCube.msr2 from testCube where "
+ + twoDaysRange;
+ String q2 = "SELECT cityid, testCube.msr2 * testCube.msr2 from testCube where "
+ + twoDaysRange;
+ String q3 = "SELECT cityid, sum(testCube.msr2) from testCube where "
+ + twoDaysRange;
+ String q4 = "SELECT cityid, sum(testCube.msr2) from testCube where "
+ + twoDaysRange + " having testCube.msr2 > 100";
+ String q5 = "SELECT cityid, testCube.msr2 from testCube where "
+ + twoDaysRange + " having testCube.msr2 + testCube.msr2 > 100";
+ String q6 = "SELECT cityid, testCube.msr2 from testCube where "
+ + twoDaysRange + " having testCube.msr2 > 100 AND testCube.msr2 < 1000";
+ String q7 = "SELECT cityid, sum(testCube.msr2) from testCube where "
+ + twoDaysRange + " having (testCube.msr2 > 100) OR (testcube.msr2 < 100" +
+ " AND SUM(testcube.msr3) > 1000)";
+
+ String expectedq1 = getExpectedQuery(cubeName, "SELECT testcube.cityid," +
+ " sum(testCube.msr2) from ", null, "group by testcube.cityid",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ String expectedq2 = null;
+ String expectedq3 = getExpectedQuery(cubeName, "SELECT testcube.cityid," +
+ " sum(testCube.msr2) from ", null, "group by testcube.cityid",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ String expectedq4 = getExpectedQuery(cubeName, "SELECT testcube.cityid," +
+ " sum(testCube.msr2) from ", null, "group by testcube.cityid having" +
+ " sum(testCube.msr2) > 100",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ String expectedq5 = null;
+ String expectedq6 = getExpectedQuery(cubeName, "SELECT testcube.cityid," +
+ " sum(testCube.msr2) from ", null, "group by testcube.cityid having" +
+ " sum(testCube.msr2) > 100 and sum(testCube.msr2) < 1000",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
+ String expectedq7 = getExpectedQuery(cubeName, "SELECT testcube.cityid," +
+ " sum(testCube.msr2) from ", null, "group by testcube.cityid having" +
+ " sum(testCube.msr2) > 100) OR (sum(testCube.msr2) < 100 AND" +
+ " SUM(testcube.msr3) > 1000)",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY"));
String tests[] = {q1, q2, q3, q4, q5, q6, q7};
+ String expected[] = {expectedq1, expectedq2, expectedq3, expectedq4,
+ expectedq5, expectedq6, expectedq7};
- try {
- for (int i = 0; i < tests.length; i++) {
+ for (int i = 0; i < tests.length; i++) {
+ try {
String hql = driver.compileCubeQuery(tests[i]);
- System.out.println("cube hql:" + hql);
+ if (expected[i] != null) {
+ compareQueries(expected[i], hql);
+ } else {
+ Assert.assertTrue("Should not reach here", false);
+ }
+ } catch (SemanticException e) {
+ e.printStackTrace();
}
- } catch (SemanticException e) {
- e.printStackTrace();
}
-
- String q8 = "SELECT cityid, testCube.noAggrMsr FROM testCube " + timeRange;
+ String q8 = "SELECT cityid, testCube.noAggrMsr FROM testCube where "
+ + twoDaysRange;
try {
// Should throw exception in aggregate resolver because noAggrMsr does
- //not have a default aggregate defined.s
+ //not have a default aggregate defined.
String hql = driver.compileCubeQuery(q8);
Assert.assertTrue("Should not reach here: " + hql, false);
} catch (SemanticException exc) {
@@ -411,8 +724,8 @@ public class TestCubeDriver {
@Test
public void testColumnAmbiguity() throws Exception {
- String timeRange = " where time_range_in('2013-05-01', '2013-05-03')";
- String query = "SELECT ambigdim1, testCube.msr1 FROM testCube " + timeRange;
+ String query = "SELECT ambigdim1, sum(testCube.msr1) FROM testCube join" +
+ " citytable on testcube.cityid = citytable.id where " + twoDaysRange;
try {
String hql = driver.compileCubeQuery(query);
@@ -422,7 +735,9 @@ public class TestCubeDriver {
exc.printStackTrace();
}
- String q2 = "SELECT ambigdim2, testCube.msr1 FROM testCube " + timeRange;
+ String q2 = "SELECT ambigdim2 from citytable join" +
+ " statetable on citytable.stateid = statetable.id join countrytable on" +
+ " statetable.countryid = countrytable.id";
try {
String hql = driver.compileCubeQuery(q2);
Assert.assertTrue("Should not reach here: " + hql, false);
@@ -433,49 +748,13 @@ public class TestCubeDriver {
}
@Test
- public void testMissingColumnValidation() throws Exception {
- String timeRange = " where time_range_in('2013-05-01', '2013-05-03')";
-
- try {
- // this query should go through
- String q1Hql =
- driver.compileCubeQuery("SELECT cityid, msr2 from testCube " +
- timeRange);
- } catch (SemanticException exc) {
- exc.printStackTrace();
- Assert.assertTrue("Exception not expected here", false);
- }
-
- try {
- // this query should through exception because invalidMsr is invalid
- String q2Hql =
- driver.compileCubeQuery("SELECT cityid, invalidMsr from testCube " +
- timeRange);
- Assert.assertTrue("Should not reach here", false);
- } catch (SemanticException exc) {
- exc.printStackTrace();
- Assert.assertNotNull(exc);
- }
- }
-
- @Test
public void testTimeRangeValidation() throws Exception {
- String timeRange1 = " where time_range_in('2013-05-01', '2013-05-03')";
-
- try {
- String hql = driver.compileCubeQuery("SELECT cityid, testCube.msr2 from" +
- " testCube " + timeRange1);
- } catch (SemanticException exc) {
- exc.printStackTrace();
- Assert.assertTrue("Exception not expected here", false);
- }
-
- // swap to and from -
- String timeRange2 = " where time_range_in('2013-05-03', '2013-05-01')";
+ String timeRange2 = " time_range_in('" + getDateUptoHours(now)
+ + "','" + getDateUptoHours(twodaysBack) + "')";
try {
// this should throw exception because from date is after to date
- String hql = driver.compileCubeQuery("SELECT cityid, testCube.msr2 from" +
- " testCube " + timeRange2);
+ driver.compileCubeQuery("SELECT cityid, testCube.msr2 from" +
+ " testCube where " + timeRange2);
Assert.assertTrue("Should not reach here", false);
} catch (SemanticException exc) {
exc.printStackTrace();
@@ -485,40 +764,65 @@ public class TestCubeDriver {
@Test
public void testAliasReplacer() throws Exception {
- String timeRange = " where time_range_in('2013-05-01', '2013-05-03')";
-
- System.out.println("##TEST_ALIAS_REPLACER");
String queries[] = {
- "SELECT cityid, t.msr2 FROM testCube t" + timeRange,
- "SELECT cityid, msr2 FROM testCube " + timeRange + " and msr2 > 100" +
- " HAVING msr2 < 1000",
- "SELECT cityid, testCube.msr2 FROM testCube " + timeRange + " and" +
- " msr2 > 100 HAVING msr2 < 1000 ORDER BY cityid"
+ "SELECT cityid, t.msr2 FROM testCube t where " + twoDaysRange,
+ "SELECT cityid, msr2 FROM testCube where msr2 > 100 and " + twoDaysRange +
+ " HAVING msr2 < 1000",
+ "SELECT cityid, testCube.msr2 FROM testCube where msr2 > 100 and "
+ + twoDaysRange + " HAVING msr2 < 1000 ORDER BY cityid"
+ };
+
+ String expectedQueries[] = {
+ getExpectedQuery("t", "SELECT t.cityid, sum(t.msr2) FROM ", null,
+ " group by t.cityid", getWhereForDailyAndHourly2days("t",
+ "C1_testfact_HOURLY", "C1_testfact_DAILY")),
+ getExpectedQuery(cubeName, "SELECT testCube.cityid, sum(testCube.msr2)" +
+ " FROM ", " testcube.msr2 > 100 ", " group by testcube.cityid having" +
+ " sum(testCube.msr2 < 1000)", getWhereForDailyAndHourly2days(
+ cubeName, "C1_testfact_HOURLY", "C1_testfact_DAILY")),
+ getExpectedQuery(cubeName, "SELECT testCube.cityid, sum(testCube.msr2)" +
+ " FROM ", " testcube.msr2 > 100 ", " group by testcube.cityid having" +
+ " sum(testCube.msr2 < 1000) orderby testCube.cityid",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testfact_HOURLY",
+ "C1_testfact_DAILY")),
};
- for (String q : queries) {
- String hql = driver.compileCubeQuery(q);
- System.out.println("@@HQL: " + hql);
+ for (int i = 0; i < queries.length; i++) {
+ String hql = driver.compileCubeQuery(queries[i]);
+ compareQueries(expectedQueries[i], hql);
}
}
@Test
public void testFactsWithInvalidColumns() throws Exception {
String hqlQuery = driver.compileCubeQuery("select dim1, AVG(msr1)," +
- " msr2 from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " msr2 from testCube" +
+ " where " + twoDaysRange);
+ String expected = getExpectedQuery(cubeName,
+ "select testcube.dim1, avg(testcube.msr1), sum(testcube.msr2) FROM ",
+ null, " group by testcube.dim1",
+ getWhereForDailyAndHourly2days(cubeName, "C1_summary1_HOURLY",
+ "C1_summary1_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select dim1, dim2, COUNT(msr1)," +
- " SUM(msr2), msr3 from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
+ " SUM(msr2), msr3 from testCube" +
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName,
+ "select testcube.dim1, testcube,dim2, count(testcube.msr1)," +
+ " sum(testcube.msr2), max(testcube.msr3) FROM ", null,
+ " group by testcube.dim1, testcube.dim2",
+ getWhereForDailyAndHourly2days(cubeName, "C1_summary2_HOURLY",
+ "C1_summary2_DAILY"));
+ compareQueries(expected, hqlQuery);
hqlQuery = driver.compileCubeQuery("select dim1, dim2, cityid, SUM(msr1)," +
- " SUM(msr2), msr3 from testCube" +
- " where time_range_in('" + getDateUptoHours(CubeTestSetup.twodaysBack)
- + "','" + getDateUptoHours(CubeTestSetup.now) + "')");
- System.out.println("cube hql:" + hqlQuery);
-
+ " SUM(msr2), msr3 from testCube" +
+ " where " + twoDaysRange);
+ expected = getExpectedQuery(cubeName,
+ "select testcube.dim1, testcube,dim2, testcube.cityid," +
+ " sum(testcube.msr1), sum(testcube.msr2), max(testcube.msr3) FROM ",
+ null, " group by testcube.dim1, testcube.dim2, testcube.cityid",
+ getWhereForDailyAndHourly2days(cubeName, "C1_summary3_HOURLY",
+ "C1_summary3_DAILY"));
+ compareQueries(expected, hqlQuery);
}
}