You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2014/11/22 00:55:51 UTC
incubator-lens git commit: LENS-21 : Moves QueryRewriter interface
from lens-driver-jdbc to lens-server-api (Suma Shivaprasad via amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master 12cca8629 -> b58979215
LENS-21 : Moves QueryRewriter interface from lens-driver-jdbc to lens-server-api (Suma Shivaprasad via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b5897921
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b5897921
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b5897921
Branch: refs/heads/master
Commit: b58979215a45460fb91c270cbc81005c09533880
Parents: 12cca86
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Sat Nov 22 05:25:29 2014 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Sat Nov 22 05:25:29 2014 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 16 +++----
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 13 +++--
.../apache/lens/driver/jdbc/QueryRewriter.java | 48 -------------------
.../driver/jdbc/TestColumnarSQLRewriter.java | 27 +++++------
.../lens/server/api/query/QueryRewriter.java | 50 ++++++++++++++++++++
5 files changed, 75 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b5897921/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 7961b39..bfc9a35 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -37,6 +37,8 @@ import org.apache.lens.api.LensException;
import org.apache.lens.cube.parse.CubeSemanticAnalyzer;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.query.QueryRewriter;
+
import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
/**
@@ -155,7 +157,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
public ColumnarSQLRewriter() {
}
- public void setConf(Configuration conf) {
+ public void init(Configuration conf) {
this.conf = new HiveConf(conf, ColumnarSQLRewriter.class);
}
@@ -778,10 +780,10 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/*
* (non-Javadoc)
*
- * @see org.apache.lens.driver.jdbc.QueryRewriter#rewrite(org.apache.hadoop.conf.Configuration, java.lang.String)
+ * @see org.apache.lens.server.api.query.QueryRewriter#rewrite(java.lang.String, org.apache.hadoop.conf.Configuration)
*/
@Override
- public synchronized String rewrite(Configuration conf, String query) throws LensException {
+ public synchronized String rewrite(String query, Configuration conf) throws LensException {
this.query = query;
StringBuilder mergedQuery = new StringBuilder();
rewrittenQuery.setLength(0);
@@ -826,9 +828,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* Replace with underlying storage.
*
* @param tree
- * the tree
- * @param metastoreClient
- * the metastore client
+ * the AST tree
*/
protected void replaceWithUnderlyingStorage(ASTNode tree) {
if (tree == null) {
@@ -888,8 +888,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/**
* Gets the underlying db name.
*
- * @param client
- * the client
* @param table
* the table
* @return the underlying db name
@@ -904,8 +902,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/**
* Gets the underlying table name.
*
- * @param client
- * the client
* @param table
* the table
* @return the underlying table name
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b5897921/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 6e79a01..a314d29 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -34,6 +34,7 @@ import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.QueryRewriter;
import org.apache.log4j.Logger;
import java.io.IOException;
@@ -346,15 +347,15 @@ public class JDBCDriver implements LensDriver {
/*
* (non-Javadoc)
*
- * @see org.apache.lens.driver.jdbc.QueryRewriter#rewrite(org.apache.hadoop.conf.Configuration, java.lang.String)
+ * @see org.apache.lens.server.api.query.QueryRewriter#rewrite(java.lang.String, org.apache.hadoop.conf.Configuration)
*/
@Override
- public String rewrite(Configuration conf, String query) throws LensException {
+ public String rewrite(String query, Configuration queryConf) throws LensException {
return query;
}
@Override
- public void setConf(Configuration conf) {
+ public void init(Configuration rewriteConf) {
}
}
@@ -439,8 +440,6 @@ public class JDBCDriver implements LensDriver {
/**
* Gets the query rewriter.
*
- * @param conf
- * the conf
* @return the query rewriter
* @throws LensException
* the lens exception
@@ -459,7 +458,7 @@ public class JDBCDriver implements LensDriver {
LOG.error("Unable to create rewriter object", e);
throw new LensException(e);
}
- rewriter.setConf(conf);
+ rewriter.init(conf);
rewriterCache.put(queryRewriterClass, rewriter);
}
return rewriter;
@@ -511,7 +510,7 @@ public class JDBCDriver implements LensDriver {
}
QueryRewriter rewriter = getQueryRewriter();
- String rewrittenQuery = rewriter.rewrite(conf, query);
+ String rewrittenQuery = rewriter.rewrite(query, conf);
if (LOG.isDebugEnabled()) {
LOG.debug("Query: " + query + " rewrittenQuery: " + rewrittenQuery);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b5897921/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
deleted file mode 100644
index 1c319ab..0000000
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/QueryRewriter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.jdbc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.lens.api.LensException;
-
-/**
- * The Interface QueryRewriter.
- */
-public interface QueryRewriter {
-
- /**
- * Rewrite.
- *
- * @param conf
- * the conf
- * @param query
- * the query
- * @return the string
- * @throws LensException
- * the lens exception
- */
- public String rewrite(Configuration conf, String query) throws LensException;
-
- /**
- * Set conf for the rewriter
- *
- * @param conf
- */
- public void setConf(Configuration conf);
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b5897921/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index 9c2809a..07f4050 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.lens.api.LensException;
import org.apache.lens.cube.metadata.MetastoreConstants;
import org.apache.lens.cube.parse.HQLParser;
-import org.apache.lens.driver.jdbc.ColumnarSQLRewriter;
import org.apache.lens.server.api.LensConfConstants;
import org.testng.Assert;
import org.testng.annotations.AfterTest;
@@ -156,7 +155,7 @@ public class TestColumnarSQLRewriter {
*/
@BeforeTest
public void setup() throws Exception {
- qtest.setConf(conf);
+ qtest.init(conf);
List<FieldSchema> factColumns = new ArrayList<FieldSchema>();
factColumns.add(new FieldSchema("item_key", "int", ""));
@@ -231,24 +230,24 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
String query = "select count(distinct id) from location_dim";
- String actual = qtest.rewrite(conf, query);
+ String actual = qtest.rewrite(query, conf);
String expected = "select count( distinct id ) from location_dim ";
compareQueries(expected, actual);
String query2 = "select count(distinct id) from location_dim location_dim";
- String actual2 = qtest.rewrite(conf, query2);
+ String actual2 = qtest.rewrite(query2, conf);
String expected2 = "select count( distinct id ) from location_dim location_dim";
compareQueries(expected2, actual2);
String query3 = "select count(distinct location_dim.id) from global_dw.location_dim location_dim";
- String actual3 = qtest.rewrite(conf, query3);
+ String actual3 = qtest.rewrite(query3, conf);
String expected3 = "select count( distinct ( location_dim . id )) from global_dw.location_dim location_dim";
compareQueries(expected3, actual3);
String query4 = "select count(distinct location_dim.id) from global_dw.location_dim location_dim "
+ "left outer join global_dw.item_dim item_dim on location_dim.id = item_dim.id "
+ "right outer join time_dim time_dim on location_dim.id = time_dim.id ";
- String actual4 = qtest.rewrite(conf, query4);
+ String actual4 = qtest.rewrite(query4, conf);
String expected4 = "select count( distinct ( location_dim . id )) from global_dw.location_dim location_dim "
+ "right outer join time_dim time_dim on (( location_dim . id ) = ( time_dim . id )) "
+ "left outer join global_dw.item_dim item_dim on (( location_dim . id ) = ( item_dim . id ))";
@@ -281,7 +280,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String rwq = qtest.rewrite(conf, query);
+ String rwq = qtest.rewrite(query, conf);
String expected = "inner join location_dim location_dim on "
+ "((( fact . location_key ) = ( location_dim . location_key )) "
+ "and (( location_dim . location_name ) = 'test123' )) "
@@ -316,7 +315,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String rwq = qtest.rewrite(conf, query);
+ String rwq = qtest.rewrite(query, conf);
Set<String> actual = setOf(qtest.rightFilter);
Assert.assertEquals(
actual,
@@ -350,7 +349,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String rwq = qtest.rewrite(conf, query);
+ String rwq = qtest.rewrite(query, conf);
Set<String> aggrActual = setOf(qtest.aggColumn);
Set<String> expectedAggr = setOf("sum(( fact . units_sold )) as sum_fact_units_sold",
"min(( fact . dollars_sold )) as min_fact_dollars_sold",
@@ -387,7 +386,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String rwq = qtest.rewrite(conf, query);
+ String rwq = qtest.rewrite(query, conf);
String expected = "fact.time_key,fact.location_key,fact.item_key,";
String actual = qtest.factKeys.toString();
compareQueries(expected, actual);
@@ -420,7 +419,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String rwq = qtest.rewrite(conf, query);
+ String rwq = qtest.rewrite(query, conf);
String expected = "fact.time_key in ( select time_dim.time_key from time_dim where ( time_dim . time_key ) "
+ "between '2013-01-01' and '2013-01-31' ) and fact.location_key in ( select location_dim.location_key "
+ "from location_dim where (( location_dim . location_name ) = 'test123' ) ) and "
@@ -459,7 +458,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String actual = qtest.rewrite(conf, query);
+ String actual = qtest.rewrite(query, conf);
String expected = "select ( fact . time_key ), ( time_dim . day_of_week ), date(( time_dim . day )), "
+ "( item_dim . item_key ), case when (sum(sum_fact_dollars_sold) = 0 ) then 0.0 "
@@ -524,7 +523,7 @@ public class TestColumnarSQLRewriter {
SessionState.start(conf);
- String actual = qtest.rewrite(conf, query);
+ String actual = qtest.rewrite(query, conf);
String expected = "select ( fact . time_key ), ( time_dim . day_of_week ), ( time_dim . day ), "
+ "case when (sum(sum_fact_dollars_sold) = 0 ) then 0.0 else sum(sum_fact_dollars_sold) end dollars_sold "
+ "from (select fact.time_key,fact.location_key,sum(( fact . dollars_sold )) as sum_fact_dollars_sold "
@@ -585,7 +584,7 @@ public class TestColumnarSQLRewriter {
+ " left outer join mytable_3 t3 on t2.t3id = t3.id " + "WHERE A = 100";
ColumnarSQLRewriter rewriter = new ColumnarSQLRewriter();
- rewriter.setConf(conf);
+ rewriter.init(conf);
rewriter.ast = HQLParser.parseHQL(query);
rewriter.query = query;
rewriter.analyzeInternal();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b5897921/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java
new file mode 100644
index 0000000..7efa2b8
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java
@@ -0,0 +1,50 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lens.api.LensException;
+
+/**
+ * The Interface QueryRewriter.
+ */
+public interface QueryRewriter {
+
+ /**
+ * Rewrite.
+ *
+ * @param queryConf
+ * the query configuration
+ * @param query
+ * the query
+ * @return the string
+ * @throws LensException
+ * the lens exception
+ */
+ public String rewrite(String query, Configuration queryConf) throws LensException;
+
+ /**
+ * Set conf for the rewriter
+ *
+ * @param rewriteConf
+ * Configuration required for rewriter init
+ */
+ public void init(Configuration rewriteConf);
+}