You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/03/01 12:52:23 UTC
incubator-lens git commit: LENS-370 Columnar sql rewriter should use
query specific conf while looking up db for native tables (jdhok)
Repository: incubator-lens
Updated Branches:
refs/heads/master 3998b9488 -> 1f33b5025
LENS-370 Columnar sql rewriter should use query specific conf while looking up db for native tables (jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1f33b502
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1f33b502
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1f33b502
Branch: refs/heads/master
Commit: 1f33b5025bd5ada135ae689c31c7dc0bdeb37854
Parents: 3998b94
Author: jdhok <ja...@inmobi.com>
Authored: Sun Mar 1 17:22:07 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Sun Mar 1 17:22:07 2015 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 34 +++++-----
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 2 +-
.../driver/jdbc/TestColumnarSQLRewriter.java | 44 ++++++++----
.../testdata/ClassLoaderTestClass.java | 23 +++++++
.../testdata/ClassLoaderTestClass2.java | 23 +++++++
lens-driver-jdbc/testdata/DatabaseJarSerde.java | 68 +++++++++++++++++++
lens-driver-jdbc/testdata/serde.jar | Bin 0 -> 733 bytes
lens-driver-jdbc/testdata/test.jar | Bin 0 -> 726 bytes
lens-driver-jdbc/testdata/test2.jar | Bin 0 -> 700 bytes
lens-driver-jdbc/testdata/testdata2.data | 5 ++
10 files changed, 167 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index db653a7..28dd662 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -45,9 +45,6 @@ import org.antlr.runtime.CommonToken;
*/
public class ColumnarSQLRewriter implements QueryRewriter {
- /** The conf. */
- private HiveConf conf;
-
/** The clause name. */
private String clauseName = null;
@@ -167,7 +164,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
@Override
public void init(Configuration conf) {
- this.conf = new HiveConf(conf, ColumnarSQLRewriter.class);
}
public String getClause() {
@@ -825,9 +821,9 @@ public class ColumnarSQLRewriter implements QueryRewriter {
*
* @throws HiveException the hive exception
*/
- public void buildQuery() throws HiveException {
+ public void buildQuery(HiveConf queryConf) throws HiveException {
analyzeInternal();
- replaceWithUnderlyingStorage(fromAST);
+ replaceWithUnderlyingStorage(queryConf, fromAST);
replaceAliasInAST();
getFilterInJoinCond(fromAST);
getAggregateColumns(selectAST);
@@ -1034,6 +1030,8 @@ public class ColumnarSQLRewriter implements QueryRewriter {
*/
@Override
public synchronized String rewrite(String query, Configuration conf) throws LensException {
+ HiveConf queryConf = new HiveConf(conf, ColumnarSQLRewriter.class);
+ queryConf.setClassLoader(conf.getClassLoader());
this.query = query;
StringBuilder mergedQuery;
rewrittenQuery.setLength(0);
@@ -1047,7 +1045,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
for (int i = 0; i < queries.length; i++) {
LOG.info("Union Query Part " + i + " : " + queries[i]);
ast = HQLParser.parseHQL(queries[i]);
- buildQuery();
+ buildQuery(queryConf);
mergedQuery = rewrittenQuery.append(" union all ");
finalRewrittenQuery = mergedQuery.toString().substring(0, mergedQuery.lastIndexOf("union all"));
reset();
@@ -1057,7 +1055,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
LOG.info("Rewritten Query : " + queryReplacedUdf);
} else {
ast = HQLParser.parseHQL(query);
- buildQuery();
+ buildQuery(queryConf);
queryReplacedUdf = replaceUDFForDB(rewrittenQuery.toString());
LOG.info("Input Query : " + query);
LOG.info("Rewritten Query : " + queryReplacedUdf);
@@ -1080,7 +1078,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
*
* @param tree the AST tree
*/
- protected void replaceWithUnderlyingStorage(ASTNode tree) {
+ protected void replaceWithUnderlyingStorage(HiveConf queryConf, ASTNode tree) {
if (tree == null) {
return;
}
@@ -1094,8 +1092,8 @@ public class ColumnarSQLRewriter implements QueryRewriter {
ASTNode dbIdentifier = (ASTNode) tree.getChild(0);
ASTNode tableIdentifier = (ASTNode) tree.getChild(1);
String lensTable = dbIdentifier.getText() + "." + tableIdentifier.getText();
- String table = getUnderlyingTableName(lensTable);
- String db = getUnderlyingDBName(lensTable);
+ String table = getUnderlyingTableName(queryConf, lensTable);
+ String db = getUnderlyingDBName(queryConf, lensTable);
// Replace both table and db names
if ("default".equalsIgnoreCase(db)) {
@@ -1111,14 +1109,14 @@ public class ColumnarSQLRewriter implements QueryRewriter {
} else {
ASTNode tableIdentifier = (ASTNode) tree.getChild(0);
String lensTable = tableIdentifier.getText();
- String table = getUnderlyingTableName(lensTable);
+ String table = getUnderlyingTableName(queryConf, lensTable);
// Replace table name
if (StringUtils.isNotBlank(table)) {
tableIdentifier.getToken().setText(table);
}
// Add db name as a new child
- String dbName = getUnderlyingDBName(lensTable);
+ String dbName = getUnderlyingDBName(queryConf, lensTable);
if (StringUtils.isNotBlank(dbName) && !"default".equalsIgnoreCase(dbName)) {
ASTNode dbIdentifier = new ASTNode(new CommonToken(HiveParser.Identifier, dbName));
dbIdentifier.setParent(tree);
@@ -1130,7 +1128,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
}
} else {
for (int i = 0; i < tree.getChildCount(); i++) {
- replaceWithUnderlyingStorage((ASTNode) tree.getChild(i));
+ replaceWithUnderlyingStorage(queryConf, (ASTNode) tree.getChild(i));
}
}
}
@@ -1142,8 +1140,8 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* @return the underlying db name
* @throws HiveException the hive exception
*/
- String getUnderlyingDBName(String table) throws HiveException {
- Table tbl = Hive.get(this.conf).getTable(table);
+ String getUnderlyingDBName(HiveConf queryConf, String table) throws HiveException {
+ Table tbl = Hive.get(queryConf).getTable(table);
return tbl == null ? null : tbl.getProperty(LensConfConstants.NATIVE_DB_NAME);
}
@@ -1154,8 +1152,8 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* @return the underlying table name
* @throws HiveException the hive exception
*/
- String getUnderlyingTableName(String table) throws HiveException {
- Table tbl = Hive.get(this.conf).getTable(table);
+ String getUnderlyingTableName(HiveConf queryConf, String table) throws HiveException {
+ Table tbl = Hive.get(queryConf).getTable(table);
return tbl == null ? null : tbl.getProperty(LensConfConstants.NATIVE_TABLE_NAME);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 83b956e..1257092 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -567,7 +567,7 @@ public class JDBCDriver implements LensDriver {
}
checkConfigured();
String explainQuery;
- String rewrittenQuery = rewriteQuery(explainCtx.getDriverQuery(this), conf);
+ String rewrittenQuery = rewriteQuery(explainCtx.getDriverQuery(this), explainCtx.getDriverConf(this));
Configuration explainConf = new Configuration(explainCtx.getDriverConf(this));
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
false);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index c061960..f62170f 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -20,6 +20,9 @@ package org.apache.lens.driver.jdbc;
import static org.testng.Assert.*;
+import java.io.File;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.util.*;
import org.apache.lens.api.LensException;
@@ -49,6 +52,7 @@ import org.testng.annotations.Test;
public class TestColumnarSQLRewriter {
HiveConf conf = new HiveConf();
+ HiveConf queryConf = new HiveConf();
ColumnarSQLRewriter qtest = new ColumnarSQLRewriter();
/**
@@ -774,20 +778,34 @@ public class TestColumnarSQLRewriter {
*/
@Test
public void testReplaceDBName() throws Exception {
- conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false);
+ File jarDir = new File("testdata");
+ File testJarFile = new File(jarDir, "test.jar");
+ File serdeJarFile = new File(jarDir, "serde.jar");
+
+ URL[] serdeUrls = new URL[2];
+ serdeUrls[0] = new URL("file:" + testJarFile.getAbsolutePath());
+ serdeUrls[1] = new URL("file:" + serdeJarFile.getAbsolutePath());
+
+ URLClassLoader createTableClassLoader = new URLClassLoader(serdeUrls, conf.getClassLoader());
+ conf.setClassLoader(createTableClassLoader);
SessionState.start(conf);
// Create test table
Database database = new Database();
database.setName("mydb");
- Hive.get().createDatabase(database);
- createTable("mydb", "mytable", "testDB", "testTable_1");
- createTable("mydb", "mytable_2", "testDB", "testTable_2");
- createTable("default", "mytable_3", "testDB", "testTable_3");
+
+ Hive.get(conf).createDatabase(database);
+ createTable(conf, "mydb", "mytable", "testDB", "testTable_1");
+ createTable(conf, "mydb", "mytable_2", "testDB", "testTable_2");
+ createTable(conf, "default", "mytable_3", "testDB", "testTable_3");
String query = "SELECT * FROM mydb.mytable t1 JOIN mydb.mytable_2 t2 ON t1.t2id = t2.id "
+ " left outer join mytable_3 t3 on t2.t3id = t3.id " + "WHERE A = 100";
+ queryConf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false);
+ // Test fails without setting this class loader as now metastore lookup is done using queryConf
+ queryConf.setClassLoader(createTableClassLoader);
+
ColumnarSQLRewriter rewriter = new ColumnarSQLRewriter();
rewriter.init(conf);
rewriter.ast = HQLParser.parseHQL(query);
@@ -798,7 +816,7 @@ public class TestColumnarSQLRewriter {
System.out.println(joinTreeBeforeRewrite);
// Rewrite
- rewriter.replaceWithUnderlyingStorage(rewriter.fromAST);
+ rewriter.replaceWithUnderlyingStorage(queryConf, rewriter.fromAST);
String joinTreeAfterRewrite = HQLParser.getString(rewriter.fromAST);
System.out.println(joinTreeAfterRewrite);
@@ -816,7 +834,7 @@ public class TestColumnarSQLRewriter {
&& joinTreeAfterRewrite.contains("testtable_3"));
// Rewrite one more query where table and db name is not set
- createTable("mydb", "mytable_4", null, null);
+ createTable(conf, "mydb", "mytable_4", null, null);
String query2 = "SELECT * FROM mydb.mytable_4 WHERE a = 100";
rewriter.ast = HQLParser.parseHQL(query2);
rewriter.query = query2;
@@ -826,7 +844,7 @@ public class TestColumnarSQLRewriter {
System.out.println(joinTreeBeforeRewrite);
// Rewrite
- rewriter.replaceWithUnderlyingStorage(rewriter.fromAST);
+ rewriter.replaceWithUnderlyingStorage(queryConf, rewriter.fromAST);
joinTreeAfterRewrite = HQLParser.getString(rewriter.fromAST);
System.out.println(joinTreeAfterRewrite);
@@ -838,14 +856,14 @@ public class TestColumnarSQLRewriter {
database = new Database();
database.setName("examples");
Hive.get().createDatabase(database);
- createTable("examples", "mytable", "default", null);
+ createTable(conf, "examples", "mytable", "default", null);
String defaultQuery = "SELECT * FROM examples.mytable t1 WHERE A = 100";
rewriter.ast = HQLParser.parseHQL(defaultQuery);
rewriter.query = defaultQuery;
rewriter.analyzeInternal();
joinTreeBeforeRewrite = HQLParser.getString(rewriter.fromAST);
- rewriter.replaceWithUnderlyingStorage(rewriter.fromAST);
+ rewriter.replaceWithUnderlyingStorage(queryConf, rewriter.fromAST);
joinTreeAfterRewrite = HQLParser.getString(rewriter.fromAST);
assertTrue(joinTreeBeforeRewrite.contains("examples"), joinTreeBeforeRewrite);
assertFalse(joinTreeAfterRewrite.contains("examples"), joinTreeAfterRewrite);
@@ -868,9 +886,9 @@ public class TestColumnarSQLRewriter {
* @param utable the utable
* @throws Exception the exception
*/
- void createTable(String db, String table, String udb, String utable) throws Exception {
+ void createTable(HiveConf conf, String db, String table, String udb, String utable) throws Exception {
Table tbl1 = new Table(db, table);
-
+ tbl1.setSerializationLib("DatabaseJarSerde");
if (StringUtils.isNotBlank(udb)) {
tbl1.setProperty(LensConfConstants.NATIVE_DB_NAME, udb);
}
@@ -883,7 +901,7 @@ public class TestColumnarSQLRewriter {
columns.add(new FieldSchema("name", "string", "col2"));
tbl1.setFields(columns);
- Hive.get().createTable(tbl1);
+ Hive.get(conf).createTable(tbl1);
System.out.println("Created table " + table);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/ClassLoaderTestClass.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/ClassLoaderTestClass.java b/lens-driver-jdbc/testdata/ClassLoaderTestClass.java
new file mode 100644
index 0000000..b295133
--- /dev/null
+++ b/lens-driver-jdbc/testdata/ClassLoaderTestClass.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/** Test class which is added to test2.jar used in TestDatabaseResourceService */
+public class ClassLoaderTestClass {
+ public static final boolean TEST_FIELD = true;
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/ClassLoaderTestClass2.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/ClassLoaderTestClass2.java b/lens-driver-jdbc/testdata/ClassLoaderTestClass2.java
new file mode 100644
index 0000000..2c4e5a0
--- /dev/null
+++ b/lens-driver-jdbc/testdata/ClassLoaderTestClass2.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/** Test class which is added to test2.jar used in TestDatabaseResourceService */
+public class ClassLoaderTestClass2 {
+ public static final boolean TEST_FIELD = true;
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/DatabaseJarSerde.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/DatabaseJarSerde.java b/lens-driver-jdbc/testdata/DatabaseJarSerde.java
new file mode 100644
index 0000000..03caff3
--- /dev/null
+++ b/lens-driver-jdbc/testdata/DatabaseJarSerde.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeStats;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Simple serde used during test of database jar
+ */
+public class DatabaseJarSerde extends AbstractSerDe {
+ // This should load class from test.jar
+ public static final ClassLoaderTestClass testClassInstance = new ClassLoaderTestClass();
+ static {
+ System.out.println("@@@@ SUCCESSFULLY_LOADED CLASS " + DatabaseJarSerde.class);
+ }
+
+ @Override
+ public void initialize(Configuration configuration, Properties properties) throws SerDeException {
+
+ }
+
+ @Override
+ public Class<? extends Writable> getSerializedClass() {
+ return null;
+ }
+
+ @Override
+ public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException {
+ return null;
+ }
+
+ @Override
+ public SerDeStats getSerDeStats() {
+ return null;
+ }
+
+ @Override
+ public Object deserialize(Writable writable) throws SerDeException {
+ return null;
+ }
+
+ @Override
+ public ObjectInspector getObjectInspector() throws SerDeException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/serde.jar
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/serde.jar b/lens-driver-jdbc/testdata/serde.jar
new file mode 100644
index 0000000..b4c0500
Binary files /dev/null and b/lens-driver-jdbc/testdata/serde.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/test.jar
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/test.jar b/lens-driver-jdbc/testdata/test.jar
new file mode 100644
index 0000000..bd0aa60
Binary files /dev/null and b/lens-driver-jdbc/testdata/test.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/test2.jar
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/test2.jar b/lens-driver-jdbc/testdata/test2.jar
new file mode 100644
index 0000000..8f54d8d
Binary files /dev/null and b/lens-driver-jdbc/testdata/test2.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1f33b502/lens-driver-jdbc/testdata/testdata2.data
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/testdata/testdata2.data b/lens-driver-jdbc/testdata/testdata2.data
new file mode 100644
index 0000000..4d0bfcb
--- /dev/null
+++ b/lens-driver-jdbc/testdata/testdata2.data
@@ -0,0 +1,5 @@
+1one
+two123item1item2
+3\Nitem1item2
+\N
+5nothing