You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/04/03 07:06:47 UTC
svn commit: r1463825 - in /hive/branches/HIVE-4115/ql/src:
java/org/apache/hadoop/hive/ql/cube/metadata/
java/org/apache/hadoop/hive/ql/cube/parse/
test/org/apache/hadoop/hive/ql/cube/processors/
Author: amareshwari
Date: Wed Apr 3 05:06:47 2013
New Revision: 1463825
URL: http://svn.apache.org/r1463825
Log:
Add storage table resolver
Modified:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/UpdatePeriod.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContextWithStorage.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtils.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java Wed Apr 3 05:06:47 2013
@@ -120,9 +120,11 @@ public final class CubeFactTable extends
if (fmt != null) {
Calendar cal = Calendar.getInstance();
cal.setTime(fromDate);
+ System.out.println(" getPartitions fromDate :" + cal.getTime());
List<String> partitions = new ArrayList<String>();
Date dt = cal.getTime();
while (dt.compareTo(toDate) < 0) {
+ System.out.println(" getPartitions nextDate :" + cal.getTime());
String part = new SimpleDateFormat(fmt).format(cal.getTime());
System.out.println("Adding partition:" + part + " for table:" + getName());
partitions.add(part);
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/UpdatePeriod.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/UpdatePeriod.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/UpdatePeriod.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/UpdatePeriod.java Wed Apr 3 05:06:47 2013
@@ -7,7 +7,7 @@ public enum UpdatePeriod implements Name
SECONDLY (Calendar.SECOND, 1000, "yyyy-MM-dd-HH-mm-ss"),
MINUTELY (Calendar.MINUTE, 60 * SECONDLY.weight(), "yyyy-MM-dd-HH-mm"),
HOURLY (Calendar.HOUR_OF_DAY, 60 * MINUTELY.weight(), "yyyy-MM-dd-HH"),
- DAILY (Calendar.DATE, 24 * HOURLY.weight(), "yyyy-MM-dd"),
+ DAILY (Calendar.DAY_OF_MONTH, 24 * HOURLY.weight(), "yyyy-MM-dd"),
WEEKLY (Calendar.WEEK_OF_YEAR, 7 * DAILY.weight(), "yyyy-'W'ww-u"),
MONTHLY (Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"),
//QUARTERLY (Calendar.MONTH, 3 * MONTHLY.weight(), "YYYY-MM"),
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java Wed Apr 3 05:06:47 2013
@@ -46,7 +46,7 @@ public class CubeQueryContext {
private Date timeTo;
private String clauseName = null;
private Map<String, List<String>> partitionCols;
- private Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap;
+ protected Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartitionMap;
public CubeQueryContext(ASTNode ast, QB qb, HiveConf conf)
throws SemanticException {
@@ -68,6 +68,7 @@ public class CubeQueryContext {
this.timeFrom = other.timeFrom;
this.timeTo = other.timeTo;
this.partitionCols = other.partitionCols;
+ this.factPartitionMap = other.factPartitionMap;
}
private QB cloneqb() {
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContextWithStorage.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContextWithStorage.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContextWithStorage.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContextWithStorage.java Wed Apr 3 05:06:47 2013
@@ -1,9 +1,11 @@
package org.apache.hadoop.hive.ql.cube.parse;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
+import org.apache.hadoop.hive.ql.cube.metadata.Storage;
import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.QB;
@@ -13,7 +15,7 @@ public class CubeQueryContextWithStorage
private final List<String> supportedStorages;
private final boolean allStoragesSupported;
- private Map<CubeFactTable, Map<UpdatePeriod, List<String>>> storageTableMap;
+ private Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factStorageMap;
public CubeQueryContextWithStorage(ASTNode ast, QB qb,
List<String> supportedStorages) throws SemanticException {
@@ -33,9 +35,9 @@ public class CubeQueryContextWithStorage
return supportedStorages;
}
- public String getStorageFactTable() {
+ public CubeFactTable getCandidateFactTable() {
if (candidateFactTables.size() > 0) {
- return candidateFactTables.iterator().next().getName();
+ return candidateFactTables.iterator().next();
}
return null;
}
@@ -43,28 +45,75 @@ public class CubeQueryContextWithStorage
String simpleQueryFormat = "SELECT %s FROM %s WHERE %s";
String joinQueryFormat = "SELECT %s FROM %s JOIN %s WHERE %s";
- @Override
- public String toHQL() throws SemanticException {
- String fromString = getStorageFactTable();
- if (fromString == null) {
- throw new SemanticException("No valid fact table available");
+ private String toHQL(CubeFactTable fact) {
+ Map<UpdatePeriod, List<String>> storageTableMap = factStorageMap.get(fact);
+ Map<UpdatePeriod, List<String>> partColMap = factPartitionMap.get(fact);
+
+ StringBuilder query = new StringBuilder();
+ Iterator<UpdatePeriod> it = partColMap.keySet().iterator();
+ while (it.hasNext()) {
+ UpdatePeriod updatePeriod = it.next();
+ query.append(toHQL(storageTableMap.get(updatePeriod).get(0),
+ partColMap.get(updatePeriod)));
+ if (it.hasNext()) {
+ query.append(" UNION ");
+ }
}
- String selectString = HQLParser.getString(getSelectTree());
- String whereString = HQLParser.getString(getWhereTree());
+ return query.toString();
+ }
+ private String toHQL(String tableName, List<String> parts) {
+ String selectString = HQLParser.getString(getSelectTree());
+ String whereString = getWhereTree(parts);
String actualQuery = String.format(simpleQueryFormat, selectString,
- fromString,
+ tableName,
whereString);
return actualQuery;
}
- public Map<CubeFactTable, Map<UpdatePeriod, List<String>>> getStorageTableMap() {
- return storageTableMap;
+ public String getWhereTree(List<String> parts) {
+ //TODO Construct where tree with part conditions
+ String originalWhereString = HQLParser.getString(super.getWhereTree());
+ String whereWithoutTimerange = originalWhereString.substring(0,
+ originalWhereString.indexOf(TIME_RANGE_FUNC));
+ String whereWithPartCols = whereWithoutTimerange + getWherePartClause(parts);
+ return whereWithPartCols;
+ }
+
+ public String getWherePartClause(List<String> parts) {
+ StringBuilder partStr = new StringBuilder();
+ for (int i = 0; i < parts.size() - 1; i++) {
+ partStr.append(Storage.getDatePartitionKey());
+ partStr.append(" = '");
+ partStr.append(parts.get(i));
+ partStr.append("'");
+ partStr.append(" OR ");
+ }
+
+ // add the last partition
+ partStr.append(Storage.getDatePartitionKey());
+ partStr.append(" = '");
+ partStr.append(parts.get(parts.size() - 1));
+ partStr.append("'");
+ return partStr.toString();
+ }
+
+ @Override
+ public String toHQL() throws SemanticException {
+ CubeFactTable candidateFactTable = getCandidateFactTable();
+ if (candidateFactTable == null) {
+ throw new SemanticException("No valid fact table available");
+ }
+ return toHQL(candidateFactTable);
+ }
+
+ public Map<CubeFactTable, Map<UpdatePeriod, List<String>>> getFactStorageMap() {
+ return factStorageMap;
}
- public void setStorageTableMap(Map<CubeFactTable,
+ public void setFactStorageMap(Map<CubeFactTable,
Map<UpdatePeriod, List<String>>> storageTableMap) {
- this.storageTableMap = storageTableMap;
+ this.factStorageMap = storageTableMap;
}
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtils.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtils.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/DateUtils.java Wed Apr 3 05:06:47 2013
@@ -192,7 +192,7 @@ public class DateUtils {
}
if (hasFraction) {
- cal.roll(interval.calendarField(), true);
+ cal.add(interval.calendarField(), 1);
return getFloorDate(cal.getTime(), interval);
} else {
return fromDate;
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java Wed Apr 3 05:06:47 2013
@@ -46,6 +46,7 @@ public class PartitionResolver implement
void getPartitions(CubeFactTable fact, Date fromDate, Date toDate,
Map<UpdatePeriod, List<String>> partitionColMap)
throws SemanticException {
+ System.out.println("getPartitions fromDate:" + fromDate + " toDate:" + toDate);
if (fromDate.equals(toDate) || fromDate.after(toDate)) {
return;
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/StorageTableResolver.java Wed Apr 3 05:06:47 2013
@@ -1,18 +1,20 @@
package org.apache.hadoop.hive.ql.cube.parse;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
+import org.apache.hadoop.hive.ql.cube.metadata.MetastoreUtil;
+import org.apache.hadoop.hive.ql.cube.metadata.Storage;
import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
import org.apache.hadoop.hive.ql.parse.SemanticException;
public class StorageTableResolver implements ContextRewriter {
public StorageTableResolver(Configuration conf) {
- // TODO Auto-generated constructor stub
}
@Override
@@ -20,11 +22,26 @@ public class StorageTableResolver implem
throws SemanticException {
CubeQueryContextWithStorage cubeqlStorage =
(CubeQueryContextWithStorage) cubeql;
- Map<CubeFactTable, Map<UpdatePeriod, List<String>>> storageTableMap =
+ Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factStorageMap =
new HashMap<CubeFactTable, Map<UpdatePeriod,List<String>>>();
+ Map<CubeFactTable, Map<UpdatePeriod, List<String>>> factPartMap =
+ cubeql.getFactPartitionMap();
//Find candidate tables wrt supported storages
-
- cubeqlStorage.setStorageTableMap(storageTableMap);
+ for (CubeFactTable fact : factPartMap.keySet()) {
+ Map<UpdatePeriod, List<String>> storageTableMap =
+ new HashMap<UpdatePeriod, List<String>>();
+ factStorageMap.put(fact, storageTableMap);
+ Map<UpdatePeriod, List<String>> partitionColMap = factPartMap.get(fact);
+ for (UpdatePeriod updatePeriod : partitionColMap.keySet()) {
+ List<String> storageTables = new ArrayList<String>();
+ storageTableMap.put(updatePeriod, storageTables);
+ for (String storage : fact.getStorages()) {
+ storageTables.add(MetastoreUtil.getFactStorageTableName(
+ fact.getName(), updatePeriod, Storage.getPrefix(storage)));
+ }
+ }
+ }
+ cubeqlStorage.setFactStorageMap(factStorageMap);
}
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1463825&r1=1463824&r2=1463825&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java Wed Apr 3 05:06:47 2013
@@ -46,8 +46,10 @@ public class TestCubeDriver {
public void testSimpleQuery2() throws Exception {
Calendar cal = Calendar.getInstance();
Date now = cal.getTime();
+ System.out.println("Test now:" + now);
cal.add(Calendar.DAY_OF_MONTH, -2);
Date twodaysBack = cal.getTime();
+ System.out.println("Test twodaysBack:" + twodaysBack);
System.out.println("Test from:" + getDateUptoHours(twodaysBack) + " to:" + getDateUptoHours(now));
String hqlQuery = driver.compileCubeQuery("select SUM(msr2) from testCube" +
" where time_range_in('" + getDateUptoHours(twodaysBack)