You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/03/20 12:41:07 UTC
[1/2] incubator-lens git commit: LENS-421 New query getting launched
for resultsetmetadata retrieval (Sushil Mohanty via jdhok)
Repository: incubator-lens
Updated Branches:
refs/heads/master 0d04bc2ac -> 78a41c994
LENS-421 New query getting launched for resultsetmetadata retrieval (Sushil Mohanty via jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/18aea029
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/18aea029
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/18aea029
Branch: refs/heads/master
Commit: 18aea0295aa51ff8fc23eae5a48c0d6d7716b082
Parents: 0d04bc2
Author: jdhok <ja...@inmobi.com>
Authored: Fri Mar 20 17:06:20 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Fri Mar 20 17:06:20 2015 +0530
----------------------------------------------------------------------
.../apache/lens/driver/jdbc/JDBCResultSet.java | 20 +-----
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 69 ++++++++++++++++++++
2 files changed, 72 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/18aea029/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
index eefc4e1..ba2403b 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
@@ -31,7 +31,6 @@ import org.apache.lens.driver.jdbc.JDBCDriver.QueryResult;
import org.apache.lens.server.api.driver.InMemoryResultSet;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hive.service.cli.ColumnDescriptor;
@@ -206,18 +205,12 @@ public class JDBCResultSet extends InMemoryResultSet {
case Types.CHAR:
case Types.NCHAR:
- hiveType = new TypeDescriptor(Type.CHAR_TYPE);
- qualifiers = new TypeQualifiers();
- qualifiers.setCharacterMaximumLength(rsmeta.getColumnDisplaySize(index));
- hiveType.setTypeQualifiers(qualifiers);
+ hiveType = new TypeDescriptor(Type.STRING_TYPE);
break;
case Types.VARCHAR:
case Types.LONGNVARCHAR:
case Types.NVARCHAR:
- hiveType = new TypeDescriptor(Type.VARCHAR_TYPE);
- qualifiers = new TypeQualifiers();
- qualifiers.setCharacterMaximumLength(rsmeta.getColumnDisplaySize(index));
- hiveType.setTypeQualifiers(qualifiers);
+ hiveType = new TypeDescriptor(Type.STRING_TYPE);
break;
case Types.NCLOB:
@@ -240,14 +233,7 @@ public class JDBCResultSet extends InMemoryResultSet {
hiveType = new TypeDescriptor(Type.TIMESTAMP_TYPE);
break;
case Types.DECIMAL:
- hiveType = new TypeDescriptor(Type.DECIMAL_TYPE);
- qualifiers = new TypeQualifiers();
- int colPrecision = rsmeta.getPrecision(index);
- if (colPrecision > HiveDecimal.MAX_PRECISION) {
- qualifiers.setPrecision(HiveDecimal.MAX_PRECISION);
- }
- qualifiers.setScale(rsmeta.getScale(index));
- hiveType.setTypeQualifiers(qualifiers);
+ hiveType = new TypeDescriptor(Type.DOUBLE_TYPE);
break;
case Types.DOUBLE:
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/18aea029/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index cb83999..550a468 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -403,6 +403,75 @@ public class TestJdbcDriver {
}
/**
+ * Test type casting of char, varchar, nvarchar and decimal type
+ *
+ * @throws Exception
+ * the exception
+ */
+ @Test
+ public void tesDecimalCharCasting() throws Exception {
+
+ Statement stmt = null;
+ Connection conn = null;
+ try {
+ conn = driver.getConnection();
+ stmt = conn.createStatement();
+ // Create table with char, varchar, nvarchar and decimal data type
+ stmt.execute("CREATE TABLE test_casting(c1 decimal(10,2), c2 varchar(20), c3 nvarchar(20), c4 char(10))");
+ // Insert data
+ stmt.execute("INSERT INTO test_casting VALUES(34.56,'abc','def','ghi')");
+ stmt.execute("INSERT INTO test_casting VALUES(78.50,'abc1','def1','ghi1')");
+ stmt.execute("INSERT INTO test_casting VALUES(48.89,'abc2','def2','ghi2')");
+ conn.commit();
+
+ // Query
+ final String query = "SELECT * FROM test_casting";
+ QueryContext context = createQueryContext(query);
+ LensResultSet resultSet = driver.execute(context);
+ assertNotNull(resultSet);
+
+ if (resultSet instanceof InMemoryResultSet) {
+ InMemoryResultSet rs = (InMemoryResultSet) resultSet;
+ LensResultSetMetadata rsMeta = rs.getMetadata();
+ assertEquals(rsMeta.getColumns().size(), 4);
+
+ ColumnDescriptor col1 = rsMeta.getColumns().get(0);
+ assertEquals(col1.getTypeName().toLowerCase(), "double");
+ assertEquals(col1.getName(), "C1");
+
+ ColumnDescriptor col2 = rsMeta.getColumns().get(1);
+ assertEquals(col2.getTypeName().toLowerCase(), "string");
+ assertEquals(col2.getName(), "C2");
+
+ ColumnDescriptor col3 = rsMeta.getColumns().get(2);
+ assertEquals(col3.getTypeName().toLowerCase(), "string");
+ assertEquals(col3.getName(), "C3");
+
+ ColumnDescriptor col4 = rsMeta.getColumns().get(3);
+ assertEquals(col4.getTypeName().toLowerCase(), "string");
+ assertEquals(col4.getName(), "C4");
+
+
+ while (rs.hasNext()) {
+ ResultRow row = rs.next();
+ List<Object> rowObjects = row.getValues();
+ }
+ if (rs instanceof JDBCResultSet) {
+ ((JDBCResultSet) rs).close();
+ }
+ }
+
+ } finally {
+ if (stmt != null) {
+ stmt.close();
+ }
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ /**
* Test prepare.
*
* @throws Exception the exception
[2/2] incubator-lens git commit: LENS-420 Metric gauge creation in
RewriteUtil can go wrong for multiple cube query (Amareshwari via jdhok)
Posted by jd...@apache.org.
LENS-420 Metric gauge creation in RewriteUtil can go wrong for multiple cube query (Amareshwari via jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/78a41c99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/78a41c99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/78a41c99
Branch: refs/heads/master
Commit: 78a41c99405fee70cb8eed0de6c7c7139c9876ff
Parents: 18aea02
Author: jdhok <ja...@inmobi.com>
Authored: Fri Mar 20 17:10:27 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Fri Mar 20 17:10:27 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/driver/cube/RewriteUtil.java | 8 +++++---
.../java/org/apache/lens/driver/cube/TestRewriting.java | 11 +++++++++--
.../org/apache/lens/server/query/TestQueryService.java | 6 ++----
3 files changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/78a41c99/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
index 3dc66ac..8ed4070 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
@@ -203,8 +203,8 @@ public final class RewriteUtil {
return query.replaceAll("[\\n\\r]", " ").replaceAll("&&", " AND ").replaceAll("\\|\\|", " OR ").trim();
}
- private static final String REWRITE_QUERY_GAUGE = RewriteUtil.class.getCanonicalName() + "-rewriteQuery";
- private static final String TOHQL_GAUGE = RewriteUtil.class.getCanonicalName() + "-rewriteQuery-toHQL";
+ private static final String REWRITE_QUERY_GAUGE = RewriteUtil.class.getSimpleName() + "-rewriteQuery";
+ private static final String TOHQL_GAUGE = RewriteUtil.class.getSimpleName() + "-rewriteQuery-toHQL";
/**
* Rewrite query.
@@ -280,6 +280,7 @@ public final class RewriteUtil {
// We have to rewrite each sub cube query which might be present in the original
// user query. We are looping through all sub queries here.
+ int qIndex = 1;
for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
if (LOG.isDebugEnabled()) {
LOG.debug("Rewriting cube query:" + cqi.query);
@@ -292,10 +293,11 @@ public final class RewriteUtil {
// Parse and rewrite individual cube query
CubeQueryContext cqc = rewriter.rewrite(cqi.query);
MethodMetricsContext toHQLGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
- TOHQL_GAUGE);
+ qIndex + "-" + TOHQL_GAUGE);
// toHQL actually generates the rewritten query
String hqlQuery = cqc.toHQL();
toHQLGauge.markSuccess();
+ qIndex++;
if (LOG.isDebugEnabled()) {
LOG.debug("Rewritten query:" + hqlQuery);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/78a41c99/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
index 34d7dca..c64a270 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
@@ -202,8 +202,8 @@ public class TestRewriting {
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestRewriting-MockDriver-org.apache.lens.driver.cube.RewriteUtil-rewriteQuery",
- "lens.MethodMetricGauge.TestRewriting-MockDriver-org.apache.lens.driver.cube.RewriteUtil-rewriteQuery-toHQL")));
+ "lens.MethodMetricGauge.TestRewriting-MockDriver-RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestRewriting-MockDriver-1-RewriteUtil-rewriteQuery-toHQL")));
conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
q2 = "insert overwrite directory '/tmp/rewrite' cube select name from table";
@@ -280,6 +280,7 @@ public class TestRewriting {
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName() + "-multiple");
q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
@@ -288,6 +289,12 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ reg = LensMetricsRegistry.getStaticRegistry();
+ Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
+ "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-1-RewriteUtil-rewriteQuery-toHQL",
+ "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-2-RewriteUtil-rewriteQuery-toHQL",
+ "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-RewriteUtil-rewriteQuery")));
+ conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
q2 = "select * from (cube select name from table) a full outer join"
+ " (cube select name2 from table2) b on a.name=b.name2";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/78a41c99/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 4c84c8b..14a4eb2 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -1541,12 +1541,10 @@ public class TestQueryService extends LensJerseyTest {
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-CUBE_REWRITE",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-DRIVER_ESTIMATE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-"
- + "org.apache.lens.driver.cube.RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-RewriteUtil-rewriteQuery",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-CUBE_REWRITE",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-DRIVER_ESTIMATE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-"
- + "org.apache.lens.driver.cube.RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-RewriteUtil-rewriteQuery",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")),
reg.getGauges().keySet().toString());
}