You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/13 11:15:41 UTC
svn commit: r1481742 - in /hive/branches/HIVE-4115/ql/src:
java/org/apache/hadoop/hive/ql/cube/metadata/
java/org/apache/hadoop/hive/ql/cube/parse/
test/org/apache/hadoop/hive/ql/cube/metadata/
test/org/apache/hadoop/hive/ql/cube/parse/ test/org/apache...
Author: amareshwari
Date: Mon May 13 09:15:40 2013
New Revision: 1481742
URL: http://svn.apache.org/r1481742
Log:
Add valid columns to fact tables; and LightestFactResolver
Added:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LightestFactResolver.java
Modified:
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreConstants.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreUtil.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/metadata/TestCubeMetastoreClient.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/CubeFactTable.java Mon May 13 09:15:40 2013
@@ -2,6 +2,7 @@ package org.apache.hadoop.hive.ql.cube.m
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
@@ -14,6 +15,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod.UpdatePeriodComparator;
import org.apache.hadoop.hive.ql.cube.parse.DateUtil;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.util.StringUtils;
public final class CubeFactTable extends AbstractCubeTable {
private final String cubeName;
@@ -200,4 +202,10 @@ public final class CubeFactTable extends
return cubeName;
}
+ public List<String> getValidColumns() {
+ String validColsStr = getProperties().get(MetastoreUtil.getValidColumnsKey(
+ getName()));
+ return validColsStr == null ? null : Arrays.asList(StringUtils.split(
+ validColsStr.toLowerCase()));
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreConstants.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreConstants.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreConstants.java Mon May 13 09:15:40 2013
@@ -14,6 +14,7 @@ public interface MetastoreConstants {
public static final String FACT_KEY_PFX = "cube.fact.";
public static final String UPDATE_PERIOD_SFX = ".updateperiods";
public static final String CUBE_NAME_SFX = ".cubename";
+ public static final String VALID_COLUMNS_SFX = ".valid.columns";
// column constants
public static final String TYPE_SFX = ".type";
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreUtil.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreUtil.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/metadata/MetastoreUtil.java Mon May 13 09:15:40 2013
@@ -166,6 +166,10 @@ public class MetastoreUtil implements Me
return getFactKeyPrefix(name) + CUBE_NAME_SFX;
}
+ public static String getValidColumnsKey(String name) {
+ return getFactKeyPrefix(name) + VALID_COLUMNS_SFX;
+ }
+
public static String getCubeTableWeightKey(String name) {
return getCubeTableKeyPrefix(name) + WEIGHT_KEY_SFX;
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/AggregateResolver.java Mon May 13 09:15:40 2013
@@ -5,6 +5,8 @@ import static org.apache.hadoop.hive.ql.
import org.antlr.runtime.CommonToken;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.cube.metadata.CubeMeasure;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -12,16 +14,17 @@ import org.apache.hadoop.hive.ql.parse.A
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.log4j.Logger;
/**
* <p>
- * Replace select and having columns with default aggregate functions on them, if default aggregate
- * is defined and if there isn't already an aggregate function specified on the columns.
+ * Replace select and having columns with default aggregate functions on them,
+ * if default aggregate is defined and if there isn't already an aggregate
+ * function specified on the columns.
* </p>
*
* <p>
- * Expressions which already contain aggregate sub-expressions will not be changed.
+ * Expressions which already contain aggregate sub-expressions will not be
+ * changed.
* </p>
*
* <p>
@@ -29,7 +32,8 @@ import org.apache.log4j.Logger;
* </p>
*/
public class AggregateResolver implements ContextRewriter {
- public static final Logger LOG = Logger.getLogger(AggregateResolver.class);
+ public static final Log LOG = LogFactory.getLog(
+ AggregateResolver.class.getName());
private final Configuration conf;
@@ -45,16 +49,13 @@ public class AggregateResolver implement
validateAggregates(cubeql, cubeql.getSelectAST(), false, false, false);
validateAggregates(cubeql, cubeql.getHavingAST(), false, false, false);
- System.out.println("Before aggregate resolver: " + cubeql.getSelectTree());
String rewritSelect = resolveForSelect(cubeql, cubeql.getSelectTree());
- System.out.println("New select after aggregate resolver: " + rewritSelect);
cubeql.setSelectTree(rewritSelect);
String rewritHaving = resolveForHaving(cubeql);
if (StringUtils.isNotBlank(rewritHaving)) {
cubeql.setHavingTree(rewritHaving);
}
- System.out.println("New having after aggregate resolver: " + rewritHaving);
}
private void validateAggregates(CubeQueryContext cubeql, ASTNode node,
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryContext.java Mon May 13 09:15:40 2013
@@ -20,6 +20,8 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.cube.metadata.AbstractCubeTable;
import org.apache.hadoop.hive.ql.cube.metadata.Cube;
@@ -44,7 +46,7 @@ public class CubeQueryContext {
public static final String TIME_RANGE_FUNC = "time_range_in";
public static final String NOW = "now";
public static final String DEFAULT_TABLE = "_default_";
-
+ public static Log LOG = LogFactory.getLog(CubeQueryContext.class.getName());
private final ASTNode ast;
private final QB qb;
private String clauseName = null;
@@ -449,12 +451,22 @@ public class CubeQueryContext {
i.hasNext();) {
CubeFactTable fact = i.next();
List<String> factCols = cubeTabToCols.get(fact);
+ List<String> validFactCols = fact.getValidColumns();
for (String col : cubeColumnsQueried) {
if (!factCols.contains(col.toLowerCase())) {
- System.out.println("Not considering the fact table:" + fact +
+ LOG.info("Not considering the fact table:" + fact +
" as column " + col + " is not available");
i.remove();
break;
+ } else {
+ if (validFactCols != null) {
+ if (!validFactCols.contains(col.toLowerCase())) {
+ LOG.info("Not considering the fact table:" + fact +
+ " as column " + col + " is not valid");
+ i.remove();
+ break;
+ }
+ }
}
}
}
@@ -593,7 +605,7 @@ public class CubeQueryContext {
QBJoinTree joinTree = qb.getQbJoinTree();
printJoinTree(joinTree, builder);
}
- System.out.println(builder.toString());
+ LOG.info(builder.toString());
}
void printJoinTree(QBJoinTree joinTree, StringBuilder builder) {
@@ -750,11 +762,6 @@ public class CubeQueryContext {
}
} else {
StringBuilder builder = new StringBuilder();
- /*
- * printJoinTree(qb.getQbJoinTree(), builder);
- * System.out.println(builder.toString());
- * builder = new StringBuilder();
- */
getQLString(qb.getQbJoinTree(), builder);
fromString = builder.toString();
}
@@ -815,7 +822,7 @@ public class CubeQueryContext {
private String toHQL(String tableName) throws SemanticException {
String qfmt = getQueryFormat();
- System.out.println("qfmt:" + qfmt);
+ LOG.info("qfmt:" + qfmt);
return String.format(qfmt, getQueryTreeStrings(tableName));
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/CubeQueryRewriter.java Mon May 13 09:15:40 2013
@@ -37,8 +37,8 @@ public class CubeQueryRewriter {
private void setupPhase2Rewriters() {
phase2Rewriters.add(new StorageTableResolver(conf));
phase2Rewriters.add(new LeastPartitionResolver(conf));
+ phase2Rewriters.add(new LightestFactResolver(conf));
phase2Rewriters.add(new LeastDimensionResolver(conf));
- // phase2Rewriters.add(new WhereConditionRewriter(conf));
}
public CubeQueryContext rewritePhase1(ASTNode astnode)
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastDimensionResolver.java Mon May 13 09:15:40 2013
@@ -5,11 +5,15 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
import org.apache.hadoop.hive.ql.parse.SemanticException;
public class LeastDimensionResolver implements ContextRewriter {
+ public static final Log LOG = LogFactory.getLog(
+ LeastDimensionResolver.class.getName());
public LeastDimensionResolver(Configuration conf) {
}
@@ -32,7 +36,7 @@ public class LeastDimensionResolver impl
cubeql.getCandidateFactTables().iterator(); i.hasNext();) {
CubeFactTable fact = i.next();
if (dimWeightMap.get(fact) > minWeight) {
- System.out.println("Removing fact:" + fact +
+ LOG.info("Removing fact:" + fact +
" from candidate fact tables as it has more dimension weight:"
+ dimWeightMap.get(fact) + " minimum:"
+ minWeight);
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LeastPartitionResolver.java Mon May 13 09:15:40 2013
@@ -6,12 +6,16 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
import org.apache.hadoop.hive.ql.parse.SemanticException;
public class LeastPartitionResolver implements ContextRewriter {
+ public static final Log LOG = LogFactory.getLog(
+ LeastPartitionResolver.class.getName());
public LeastPartitionResolver(Configuration conf) {
}
@@ -35,7 +39,7 @@ public class LeastPartitionResolver impl
cubeql.getCandidateFactTables().iterator(); i.hasNext();) {
CubeFactTable fact = i.next();
if (numPartitionsMap.get(fact) > minPartitions) {
- System.out.println("Removing fact:" + fact +
+ LOG.info("Removing fact:" + fact +
" from candidate fact tables as it requires more partitions to" +
" be queried:" + numPartitionsMap.get(fact) + " minimum:"
+ minPartitions);
Added: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LightestFactResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LightestFactResolver.java?rev=1481742&view=auto
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LightestFactResolver.java (added)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/LightestFactResolver.java Mon May 13 09:15:40 2013
@@ -0,0 +1,47 @@
+package org.apache.hadoop.hive.ql.cube.parse;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+public class LightestFactResolver implements ContextRewriter {
+ public static final Log LOG = LogFactory.getLog(
+ LightestFactResolver.class.getName());
+
+ public LightestFactResolver(Configuration conf) {
+ }
+
+ @Override
+ public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
+ if (cubeql.getCube() != null && !cubeql.getCandidateFactTables()
+ .isEmpty()) {
+ Map<CubeFactTable, Double> factWeightMap =
+ new HashMap<CubeFactTable, Double>();
+
+ for (CubeFactTable fact : cubeql.getCandidateFactTables()) {
+ factWeightMap.put(fact, fact.weight());
+ }
+
+ double minWeight = Collections.min(factWeightMap.values());
+
+ for (Iterator<CubeFactTable> i =
+ cubeql.getCandidateFactTables().iterator(); i.hasNext();) {
+ CubeFactTable fact = i.next();
+ if (factWeightMap.get(fact) > minWeight) {
+ LOG.info("Removing fact:" + fact +
+ " from candidate fact tables as it has more fact weight:"
+ + factWeightMap.get(fact) + " minimum:"
+ + minWeight);
+ i.remove();
+ }
+ }
+ }
+ }
+}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/cube/parse/PartitionResolver.java Mon May 13 09:15:40 2013
@@ -8,12 +8,16 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
import org.apache.hadoop.hive.ql.cube.metadata.UpdatePeriod;
import org.apache.hadoop.hive.ql.parse.SemanticException;
public class PartitionResolver implements ContextRewriter {
+ public static final Log LOG = LogFactory.getLog(
+ PartitionResolver.class.getName());
public PartitionResolver(Configuration conf) {
}
@@ -53,7 +57,7 @@ public class PartitionResolver implement
UpdatePeriod interval = fact.maxIntervalInRange(fromDate, toDate);
if (interval == null) {
- System.err.println("Could not find partition for given range:"
+ LOG.info("Could not find partition for given range:"
+ fromDate + "-" + toDate + " in fact:" + fact.getName());
return false;
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/metadata/TestCubeMetastoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/metadata/TestCubeMetastoreClient.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/metadata/TestCubeMetastoreClient.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/metadata/TestCubeMetastoreClient.java Mon May 13 09:15:40 2013
@@ -29,7 +29,7 @@ public class TestCubeMetastoreClient {
private Cube cube;
private Set<CubeMeasure> cubeMeasures;
private Set<CubeDimension> cubeDimensions;
- private final String cubeName = "testCube";
+ private final String cubeName = "testMetastoreCube";
private Date now;
@Before
@@ -104,7 +104,7 @@ public class TestCubeMetastoreClient {
@Test
public void testCubeFact() throws Exception {
- String factName = "testFact";
+ String factName = "testMetastoreFact";
List<FieldSchema> factColumns = new ArrayList<FieldSchema>(
cubeMeasures.size());
for (CubeMeasure measure : cubeMeasures) {
@@ -398,7 +398,7 @@ public class TestCubeMetastoreClient {
@Test
public void testCubeDim() throws Exception {
- String dimName = "ziptable";
+ String dimName = "ziptableMeta";
List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
dimColumns.add(new FieldSchema("zipcode", "int", "code"));
@@ -444,7 +444,7 @@ public class TestCubeMetastoreClient {
@Test
public void testCubeDimWithoutDumps() throws Exception {
- String dimName = "countrytable";
+ String dimName = "countrytableMeta";
List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
dimColumns.add(new FieldSchema("id", "int", "code"));
@@ -484,7 +484,7 @@ public class TestCubeMetastoreClient {
@Test
public void testCubeDimWithTwoStorages() throws Exception {
- String dimName = "citytable";
+ String dimName = "citytableMeta";
List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
dimColumns.add(new FieldSchema("id", "int", "code"));
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/parse/CubeTestSetup.java Mon May 13 09:15:40 2013
@@ -14,12 +14,14 @@ import org.apache.hadoop.hive.ql.cube.me
import org.apache.hadoop.hive.ql.cube.metadata.ColumnMeasure;
import org.apache.hadoop.hive.ql.cube.metadata.Cube;
import org.apache.hadoop.hive.ql.cube.metadata.CubeDimension;
+import org.apache.hadoop.hive.ql.cube.metadata.CubeFactTable;
import org.apache.hadoop.hive.ql.cube.metadata.CubeMeasure;
import org.apache.hadoop.hive.ql.cube.metadata.CubeMetastoreClient;
import org.apache.hadoop.hive.ql.cube.metadata.ExprMeasure;
import org.apache.hadoop.hive.ql.cube.metadata.HDFSStorage;
import org.apache.hadoop.hive.ql.cube.metadata.HierarchicalDimension;
import org.apache.hadoop.hive.ql.cube.metadata.InlineDimension;
+import org.apache.hadoop.hive.ql.cube.metadata.MetastoreUtil;
import org.apache.hadoop.hive.ql.cube.metadata.ReferencedDimension;
import org.apache.hadoop.hive.ql.cube.metadata.Storage;
import org.apache.hadoop.hive.ql.cube.metadata.TableReference;
@@ -82,8 +84,6 @@ public class CubeTestSetup {
cubeDimensions.add(new ReferencedDimension(
new FieldSchema("dim2", "string", "ref dim"),
new TableReference("testdim2", "id")));
- cubeDimensions.add(new InlineDimension(
- new FieldSchema("region", "string", "region dim"), regions));
cube = new Cube(cubeName, cubeMeasures, cubeDimensions);
client.createCube(cubeName, cubeMeasures, cubeDimensions);
}
@@ -286,6 +286,82 @@ public class CubeTestSetup {
createZiptable(client);
createCountryTable(client);
createStateTable(client);
+ createCubeFactsWithValidColumns(client);
+ }
+
+ private void createCubeFactsWithValidColumns(CubeMetastoreClient client)
+ throws HiveException {
+ String factName = "summary1";
+ StringBuilder commonCols = new StringBuilder();
+ List<FieldSchema> factColumns = new ArrayList<FieldSchema>(
+ cubeMeasures.size());
+ for (CubeMeasure measure : cubeMeasures) {
+ factColumns.add(measure.getColumn());
+ commonCols.append(measure.getName());
+ commonCols.append(",");
+ }
+
+ // add dimensions of the cube
+ factColumns.add(new FieldSchema("dim1","string", "dim1"));
+ factColumns.add(new FieldSchema("dim2","string", "dim2"));
+ factColumns.add(new FieldSchema("zipcode","int", "zip"));
+ factColumns.add(new FieldSchema("cityid","int", "city id"));
+ List<UpdatePeriod> updates = new ArrayList<UpdatePeriod>();
+ updates.add(UpdatePeriod.MINUTELY);
+ updates.add(UpdatePeriod.HOURLY);
+ updates.add(UpdatePeriod.DAILY);
+ Map<String, List<UpdatePeriod>> storageUpdatePeriods =
+ new HashMap<String, List<UpdatePeriod>>();
+
+ Map<Storage, List<UpdatePeriod>> storageAggregatePeriods =
+ new HashMap<Storage, List<UpdatePeriod>>();
+ Storage hdfsStorage = new HDFSStorage("C1",
+ TextInputFormat.class.getCanonicalName(),
+ HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ storageAggregatePeriods.put(hdfsStorage, updates);
+ storageUpdatePeriods.put(hdfsStorage.getName(), updates);
+
+ // create cube fact summary1
+ Map<String, String> properties = new HashMap<String, String>();
+ String validColumns = commonCols.toString() + ",dim1";
+ properties.put(MetastoreUtil.getValidColumnsKey(factName),
+ validColumns);
+ CubeFactTable fact1 = new CubeFactTable(cubeName, factName, factColumns,
+ storageUpdatePeriods, 10L, properties);
+ client.createCubeTable(fact1, storageAggregatePeriods);
+
+ // create summary2 - same schema, different valid columns
+ factName = "summary2";
+ storageAggregatePeriods =
+ new HashMap<Storage, List<UpdatePeriod>>();
+ hdfsStorage = new HDFSStorage("C1",
+ TextInputFormat.class.getCanonicalName(),
+ HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ storageAggregatePeriods.put(hdfsStorage, updates);
+ storageUpdatePeriods.put(hdfsStorage.getName(), updates);
+ properties = new HashMap<String, String>();
+ validColumns = commonCols.toString() + ",dim1,dim2";
+ properties.put(MetastoreUtil.getValidColumnsKey(factName),
+ validColumns);
+ CubeFactTable fact2 = new CubeFactTable(cubeName, factName, factColumns,
+ storageUpdatePeriods, 20L, properties);
+ client.createCubeTable(fact2, storageAggregatePeriods);
+
+ factName = "summary3";
+ storageAggregatePeriods =
+ new HashMap<Storage, List<UpdatePeriod>>();
+ hdfsStorage = new HDFSStorage("C1",
+ TextInputFormat.class.getCanonicalName(),
+ HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ storageAggregatePeriods.put(hdfsStorage, updates);
+ storageUpdatePeriods.put(hdfsStorage.getName(), updates);
+ properties = new HashMap<String, String>();
+ validColumns = commonCols.toString() + ",dim1,dim2,cityid";
+ properties.put(MetastoreUtil.getValidColumnsKey(factName),
+ validColumns);
+ CubeFactTable fact3 = new CubeFactTable(cubeName, factName, factColumns,
+ storageUpdatePeriods, 30L, properties);
+ client.createCubeTable(fact3, storageAggregatePeriods);
}
}
Modified: hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java?rev=1481742&r1=1481741&r2=1481742&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java (original)
+++ hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/cube/processors/TestCubeDriver.java Mon May 13 09:15:40 2013
@@ -78,7 +78,7 @@ public class TestCubeDriver {
public void testCandidateTables() throws Exception {
Throwable th = null;
try {
- String hqlQuery = driver.compileCubeQuery("select dim1, SUM(msr2)" +
+ String hqlQuery = driver.compileCubeQuery("select dim12, SUM(msr2)" +
" from testCube" +
" where time_range_in('" + getDateUptoHours(twodaysBack)
+ "','" + getDateUptoHours(now) + "')");
@@ -401,7 +401,7 @@ public class TestCubeDriver {
String hql = driver.compileCubeQuery("SELECT cityid, testCube.msr2 from" +
" testCube " + timeRange1);
} catch (SemanticException exc) {
- exc.printStackTrace(System.out);
+ exc.printStackTrace();
Assert.assertTrue("Exception not expected here", false);
}
@@ -413,7 +413,7 @@ public class TestCubeDriver {
" testCube " + timeRange2);
Assert.assertTrue("Should not reach here", false);
} catch (SemanticException exc) {
- exc.printStackTrace(System.out);
+ exc.printStackTrace();
Assert.assertNotNull(exc);
}
}
@@ -425,19 +425,35 @@ public class TestCubeDriver {
System.out.println("##TEST_ALIAS_REPLACER");
String queries[] = {
"SELECT cityid, t.msr2 FROM testCube t" + timeRange,
- "SELECT cityid, msr2 FROM testCube " + timeRange + " and msr2 > 100 HAVING msr2 < 1000",
- "SELECT cityid, testCube.msr2 FROM tetCube " + timeRange + " and msr2 > 100 HAVING msr2 < 1000 ORDER BY cityid"
+ "SELECT cityid, msr2 FROM testCube " + timeRange + " and msr2 > 100" +
+ " HAVING msr2 < 1000",
+ "SELECT cityid, testCube.msr2 FROM testCube " + timeRange + " and" +
+ " msr2 > 100 HAVING msr2 < 1000 ORDER BY cityid"
};
- try {
- for (String q : queries) {
- String hql = driver.compileCubeQuery(q);
- System.out.println("@@QUERY: " + q);
- System.out.println("@@HQL: " + hql);
- }
- } catch (Exception exc) {
- exc.printStackTrace();
+ for (String q : queries) {
+ String hql = driver.compileCubeQuery(q);
+ System.out.println("@@HQL: " + hql);
}
}
+ @Test
+ public void testFactsWithInvalidColumns() throws Exception {
+ String hqlQuery = driver.compileCubeQuery("select dim1, AVG(msr1)," +
+ " msr2 from testCube" +
+ " where time_range_in('" + getDateUptoHours(twodaysBack)
+ + "','" + getDateUptoHours(now) + "')");
+ System.out.println("cube hql:" + hqlQuery);
+ hqlQuery = driver.compileCubeQuery("select dim1, dim2, COUNT(msr1), SUM(msr2)," +
+ " msr3 from testCube" +
+ " where time_range_in('" + getDateUptoHours(twodaysBack)
+ + "','" + getDateUptoHours(now) + "')");
+ System.out.println("cube hql:" + hqlQuery);
+ hqlQuery = driver.compileCubeQuery("select dim1, dim2, cityid, SUM(msr1)," +
+ " SUM(msr2), msr3 from testCube" +
+ " where time_range_in('" + getDateUptoHours(twodaysBack)
+ + "','" + getDateUptoHours(now) + "')");
+ System.out.println("cube hql:" + hqlQuery);
+
+ }
}