You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/04/18 13:35:52 UTC
[kylin] branch master updated: KYLIN-3857 add configuration for
quote character
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 426a227 KYLIN-3857 add configuration for quote character
426a227 is described below
commit 426a22711d6e25d2b7a4c1c3c6cea4e642b7f248
Author: hit-lacus <hi...@126.com>
AuthorDate: Wed Apr 17 20:57:43 2019 +0800
KYLIN-3857 add configuration for quote character
---
.../org/apache/kylin/common/KylinConfigBase.java | 8 +++
.../java/org/apache/kylin/job/JoinedFlatTable.java | 13 ++--
.../kylin/job/util/FlatTableSqlQuoteUtils.java | 81 +++++++++++-----------
.../kylin/job/util/FlatTableSqlQuoteUtilsTest.java | 35 ++++++----
.../kylin/source/hive/GarbageCollectionStep.java | 8 ++-
.../apache/kylin/source/hive/HiveMRInputTest.java | 9 ++-
.../kylin/source/jdbc/JdbcHiveInputBase.java | 5 +-
.../source/jdbc/extensible/JdbcHiveInputBase.java | 5 +-
.../apache/kylin/source/kafka/KafkaInputBase.java | 4 +-
9 files changed, 98 insertions(+), 70 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 849f635..e8d9480 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -873,6 +873,14 @@ public abstract class KylinConfigBase implements Serializable {
return r;
}
+ public boolean enableHiveDdlQuote(){
+ return Boolean.parseBoolean(getOptional("kylin.source.hive.quote-enabled", TRUE));
+ }
+
+ public String getQuoteCharacter(){
+ return getOptional("kylin.source.quote.character", "`");
+ }
+
/**
* was for route to hive, not used any more
*/
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 297f669..427f379 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -32,6 +32,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.util.FlatTableSqlQuoteUtils;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.JoinDesc;
@@ -89,7 +90,7 @@ public class JoinedFlatTable {
if (i > 0) {
ddl.append(",");
}
- ddl.append(colName(col, flatDesc.useAlias()) + " " + getHiveDataType(col.getDatatype()) + "\n");
+ ddl.append(quoteIdentifier(colName(col, flatDesc.useAlias())) + " " + getHiveDataType(col.getDatatype()) + "\n");
}
ddl.append(")" + "\n");
if (TEXTFILE.equals(storageFormat)) {
@@ -170,7 +171,7 @@ public class JoinedFlatTable {
DataModelDesc model = flatDesc.getDataModel();
TableRef rootTable = model.getRootFactTable();
- sql.append(" FROM ").append(flatDesc.getDataModel().getRootFactTable().getTableIdentityQuoted("`"))
+ sql.append(" FROM ").append(flatDesc.getDataModel().getRootFactTable().getTableIdentityQuoted(FlatTableSqlQuoteUtils.getQuote()))
.append(" as ").append(quoteIdentifier(rootTable.getAlias())).append(sep);
for (JoinTableDesc lookupDesc : model.getJoinTables()) {
@@ -185,7 +186,7 @@ public class JoinedFlatTable {
}
String joinType = join.getType().toUpperCase(Locale.ROOT);
- sql.append(joinType).append(" JOIN ").append(dimTable.getTableIdentityQuoted("`"))
+ sql.append(joinType).append(" JOIN ").append(dimTable.getTableIdentityQuoted(FlatTableSqlQuoteUtils.getQuote()))
.append(" as ").append(quoteIdentifier(dimTable.getAlias())).append(sep);
sql.append("ON ");
for (int i = 0; i < pk.length; i++) {
@@ -229,7 +230,7 @@ public class JoinedFlatTable {
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc,
- model.getFilterCondition(), "`");
+ model.getFilterCondition());
whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
}
if (flatDesc.getSegment() != null) {
@@ -240,7 +241,7 @@ public class JoinedFlatTable {
if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
- partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange), "`");
+ partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange));
whereBuilder.append(quotedPartitionCond);
whereBuilder.append(")" + sep);
}
@@ -313,7 +314,7 @@ public class JoinedFlatTable {
+ quoteIdentifier(col.getName());
} else {
String computeExpr = col.getColumnDesc().getComputedColumnExpr();
- return quoteIdentifierInSqlExpr(flatDesc, computeExpr, "`");
+ return quoteIdentifierInSqlExpr(flatDesc, computeExpr);
}
}
}
diff --git a/core-job/src/main/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtils.java b/core-job/src/main/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtils.java
index 2d0f72d..1b7073a 100644
--- a/core-job/src/main/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtils.java
+++ b/core-job/src/main/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtils.java
@@ -26,6 +26,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -36,47 +37,50 @@ import com.google.common.collect.Maps;
public class FlatTableSqlQuoteUtils {
- public static final String QUOTE = "`";
+ private FlatTableSqlQuoteUtils() {
+ }
+
+ private static String quote = null;
+
+ private static synchronized void setQuote() {
+ if (quote != null)
+ return;
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ if (kylinConfig.enableHiveDdlQuote()) {
+ quote = kylinConfig.getQuoteCharacter();
+ } else {
+ quote = "";
+ }
+ }
+
+ public static String getQuote() {
+ setQuote();
+ return quote;
+ }
/**
* Quote identifier by default quote `
- * @param identifier
- * @return
*/
- public static String quoteIdentifier(String identifier){
- return QUOTE + identifier + QUOTE;
+ public static String quoteIdentifier(String identifier) {
+ setQuote();
+ return quote + identifier + quote;
}
/**
* Quote table identity, eg. `default`.`kylin_sales`
- * @param database
- * @param table
- * @param quote
- * @return
*/
- public static String quoteTableIdentity(String database, String table, String quote) {
+ public static String quoteTableIdentity(String database, String table) {
+ setQuote();
String dbName = quote + database + quote;
String tableName = quote + table + quote;
return String.format(Locale.ROOT, "%s.%s", dbName, tableName).toUpperCase(Locale.ROOT);
}
/**
- * use default quote ` to quote table identity
- * @param database
- * @param table
- * @return
- */
- public static String quoteTableIdentity(String database, String table) {
- return quoteTableIdentity(database, table, QUOTE);
- }
-
- /**
* Used for quote identifiers in Sql Filter Expression & Computed Column Expression for flat table
- * @param flatDesc
- * @param quotation
- * @return
*/
- public static String quoteIdentifierInSqlExpr(IJoinedFlatTableDesc flatDesc, String sqlExpr, String quotation) {
+ public static String quoteIdentifierInSqlExpr(IJoinedFlatTableDesc flatDesc, String sqlExpr) {
+ setQuote();
Map<String, String> tabToAliasMap = buildTableToTableAliasMap(flatDesc);
Map<String, Map<String, String>> tabToColsMap = buildTableToColumnsMap(flatDesc);
@@ -84,14 +88,14 @@ public class FlatTableSqlQuoteUtils {
for (String table : tabToAliasMap.keySet()) {
List<String> tabPatterns = getTableNameOrAliasPatterns(table);
if (isIdentifierNeedToQuote(sqlExpr, table, tabPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, table, tabPatterns);
+ sqlExpr = quoteIdentifier(sqlExpr, table, tabPatterns);
tableMatched = true;
}
String tabAlias = tabToAliasMap.get(table);
List<String> tabAliasPatterns = getTableNameOrAliasPatterns(tabAlias);
if (isIdentifierNeedToQuote(sqlExpr, tabAlias, tabAliasPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, tabAlias, tabAliasPatterns);
+ sqlExpr = quoteIdentifier(sqlExpr, tabAlias, tabAliasPatterns);
tableMatched = true;
}
@@ -100,13 +104,13 @@ public class FlatTableSqlQuoteUtils {
for (String column : columns) {
List<String> colPatterns = getColumnNameOrAliasPatterns(column);
if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, column, colPatterns);
+ sqlExpr = quoteIdentifier(sqlExpr, column, colPatterns);
}
if (columnHasAlias(table, column, tabToColsMap)) {
String colAlias = getColumnAlias(table, column, tabToColsMap);
List<String> colAliasPattern = getColumnNameOrAliasPatterns(colAlias);
if (isIdentifierNeedToQuote(sqlExpr, colAlias, colAliasPattern)) {
- sqlExpr = quoteIdentifier(sqlExpr, quotation, colAlias, colPatterns);
+ sqlExpr = quoteIdentifier(sqlExpr, colAlias, colPatterns);
}
}
}
@@ -121,15 +125,14 @@ public class FlatTableSqlQuoteUtils {
* Used to quote identifiers for JDBC ext job when quoting cc expr
* @param tableDesc
* @param sqlExpr
- * @param quot
* @return
*/
- public static String quoteIdentifierInSqlExpr(TableDesc tableDesc, String sqlExpr, String quot) {
+ public static String quoteIdentifierInSqlExpr(TableDesc tableDesc, String sqlExpr) {
String table = tableDesc.getName();
boolean tableMatched = false;
List<String> tabPatterns = getTableNameOrAliasPatterns(table);
if (isIdentifierNeedToQuote(sqlExpr, table, tabPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quot, table, tabPatterns);
+ sqlExpr = quoteIdentifier(sqlExpr, table, tabPatterns);
tableMatched = true;
}
@@ -138,7 +141,7 @@ public class FlatTableSqlQuoteUtils {
String column = columnDesc.getName();
List<String> colPatterns = getColumnNameOrAliasPatterns(column);
if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
- sqlExpr = quoteIdentifier(sqlExpr, quot, column, colPatterns);
+ sqlExpr = quoteIdentifier(sqlExpr, column, colPatterns);
}
}
}
@@ -164,9 +167,9 @@ public class FlatTableSqlQuoteUtils {
}
// visible for test
- static String quoteIdentifier(String sqlExpr, String quotation, String identifier,
- List<String> identifierPatterns) {
- String quotedIdentifier = quotation + identifier.trim() + quotation;
+ static String quoteIdentifier(String sqlExpr, String identifier, List<String> identifierPatterns) {
+ setQuote();
+ String quotedIdentifier = quote + identifier.trim() + quote;
for (String pattern : identifierPatterns) {
Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.DOTALL).matcher(sqlExpr);
@@ -220,7 +223,7 @@ public class FlatTableSqlQuoteUtils {
}
private static Map<String, String> getColToColAliasMapInTable(String tableName,
- Map<String, Map<String, String>> tableToColumnsMap) {
+ Map<String, Map<String, String>> tableToColumnsMap) {
if (tableToColumnsMap.containsKey(tableName)) {
return tableToColumnsMap.get(tableName);
}
@@ -228,13 +231,13 @@ public class FlatTableSqlQuoteUtils {
}
private static Set<String> listColumnsInTable(String tableName,
- Map<String, Map<String, String>> tableToColumnsMap) {
+ Map<String, Map<String, String>> tableToColumnsMap) {
Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
return colToAliasMap.keySet();
}
private static boolean columnHasAlias(String tableName, String columnName,
- Map<String, Map<String, String>> tableToColumnsMap) {
+ Map<String, Map<String, String>> tableToColumnsMap) {
Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
if (colToAliasMap.containsKey(columnName)) {
return true;
@@ -243,7 +246,7 @@ public class FlatTableSqlQuoteUtils {
}
private static String getColumnAlias(String tableName, String columnName,
- Map<String, Map<String, String>> tableToColumnsMap) {
+ Map<String, Map<String, String>> tableToColumnsMap) {
Map<String, String> colToAliasMap = getColToColAliasMapInTable(tableName, tableToColumnsMap);
if (colToAliasMap.containsKey(columnName)) {
return colToAliasMap.get(columnName);
diff --git a/core-job/src/test/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtilsTest.java b/core-job/src/test/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtilsTest.java
index 1b5839f..c1d41a8 100644
--- a/core-job/src/test/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtilsTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/util/FlatTableSqlQuoteUtilsTest.java
@@ -18,34 +18,41 @@
package org.apache.kylin.job.util;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import java.util.List;
-public class FlatTableSqlQuoteUtilsTest {
+public class FlatTableSqlQuoteUtilsTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ createTestMetadata();
+ }
@Test
public void testQuoteTableName() {
List<String> tablePatterns = FlatTableSqlQuoteUtils.getTableNameOrAliasPatterns("KYLIN_SALES");
String expr = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
String expectedExpr = "`KYLIN_SALES`.PRICE * `KYLIN_SALES`.COUNT";
- String quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES", tablePatterns);
+ String quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "`KYLIN_SALES`.PRICE * KYLIN_SALES.COUNT";
expectedExpr = "`KYLIN_SALES`.PRICE * `KYLIN_SALES`.COUNT";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES", tablePatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
expectedExpr = "`KYLIN_SALES`.PRICE AS KYLIN_SALES_PRICE * `KYLIN_SALES`.COUNT AS KYLIN_SALES_COUNT";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES", tablePatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "(KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
expectedExpr = "(`KYLIN_SALES`.PRICE AS KYLIN_SALES_PRICE > 1 and `KYLIN_SALES`.COUNT AS KYLIN_SALES_COUNT > 50)";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES", tablePatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
}
@@ -54,22 +61,22 @@ public class FlatTableSqlQuoteUtilsTest {
List<String> tablePatterns = FlatTableSqlQuoteUtils.getTableNameOrAliasPatterns("KYLIN_SALES_ALIAS");
String expr = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
String expectedExpr = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
- String quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES_ALIAS", tablePatterns);
+ String quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES_ALIAS", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
expectedExpr = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES_ALIAS", tablePatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES_ALIAS", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "(KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
expectedExpr = "(KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES_ALIAS", tablePatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES_ALIAS", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "(KYLIN_SALES_ALIAS.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
expectedExpr = "(`KYLIN_SALES_ALIAS`.PRICE AS KYLIN_SALES_PRICE > 1 and KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT > 50)";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "KYLIN_SALES_ALIAS", tablePatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "KYLIN_SALES_ALIAS", tablePatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
}
@@ -78,27 +85,27 @@ public class FlatTableSqlQuoteUtilsTest {
List<String> columnPatterns = FlatTableSqlQuoteUtils.getColumnNameOrAliasPatterns("PRICE");
String expr = "KYLIN_SALES.PRICE * KYLIN_SALES.COUNT";
String expectedExpr = "KYLIN_SALES.`PRICE` * KYLIN_SALES.COUNT";
- String quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "PRICE", columnPatterns);
+ String quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "PRICE", columnPatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "KYLIN_SALES.PRICE/KYLIN_SALES.COUNT";
expectedExpr = "KYLIN_SALES.`PRICE`/KYLIN_SALES.COUNT";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "PRICE", columnPatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "PRICE", columnPatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "KYLIN_SALES.PRICE AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
expectedExpr = "KYLIN_SALES.`PRICE` AS KYLIN_SALES_PRICE * KYLIN_SALES.COUNT AS KYLIN_SALES_COUNT";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "PRICE", columnPatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "PRICE", columnPatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "(PRICE > 1 AND COUNT > 50)";
expectedExpr = "(`PRICE` > 1 AND COUNT > 50)";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "PRICE", columnPatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "PRICE", columnPatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
expr = "PRICE>1 and `PRICE` < 15";
expectedExpr = "`PRICE`>1 and `PRICE` < 15";
- quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "`", "PRICE", columnPatterns);
+ quotedExpr = FlatTableSqlQuoteUtils.quoteIdentifier(expr, "PRICE", columnPatterns);
Assert.assertEquals(expectedExpr, quotedExpr);
}
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java
index ebfaf85..ed86513 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java
@@ -32,6 +32,7 @@ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
+import org.apache.kylin.job.util.FlatTableSqlQuoteUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,14 +58,17 @@ public class GarbageCollectionStep extends AbstractExecutable {
}
private String cleanUpIntermediateFlatTable(KylinConfig config) throws IOException {
+ String quoteCharacter = FlatTableSqlQuoteUtils.getQuote();
StringBuffer output = new StringBuffer();
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
final List<String> hiveTables = this.getIntermediateTables();
- if (!config.isHiveKeepFlatTable()){
+ if (!config.isHiveKeepFlatTable()) {
for (String hiveTable : hiveTables) {
if (StringUtils.isNotEmpty(hiveTable)) {
hiveCmdBuilder.addStatement("USE " + config.getHiveDatabaseForIntermediateTable() + ";");
- hiveCmdBuilder.addStatement("DROP TABLE IF EXISTS `" + hiveTable + "`;");
+ hiveCmdBuilder
+ .addStatement("DROP TABLE IF EXISTS " + quoteCharacter + hiveTable + quoteCharacter + ";");
+
output.append("Hive table " + hiveTable + " is dropped. \n");
}
}
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index cc1b5e1..d0e30a0 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -28,13 +28,20 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
-public class HiveMRInputTest {
+public class HiveMRInputTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ createTestMetadata();
+ }
@Test
public void TestGetJobWorkingDir() throws IOException {
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
index 3769473..77e38c7 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
@@ -183,8 +183,7 @@ public class JdbcHiveInputBase extends HiveInputBase {
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
- flatDesc.getSegment(), segRange),
- "`");
+ flatDesc.getSegment(), segRange));
bquery += " WHERE " + quotedPartCond;
}
}
@@ -196,7 +195,7 @@ public class JdbcHiveInputBase extends HiveInputBase {
String cmd = String.format(Locale.ROOT,
"%s/bin/sqoop import" + generateSqoopConfigArgString()
- + "--connect \"%s\" --driver %s --username %s --password %s --query \"%s AND \\$CONDITIONS\" "
+ + "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
+ "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '' "
+ "--fields-terminated-by '%s' --num-mappers %d",
sqoopHome, connectionUrl, driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable,
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
index ec69084..9fd6d30 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
@@ -94,15 +94,14 @@ public class JdbcHiveInputBase extends org.apache.kylin.source.jdbc.JdbcHiveInpu
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
- flatDesc.getSegment(), segRange),
- "`");
+ flatDesc.getSegment(), segRange));
bquery += " WHERE " + quotedPartCond;
}
}
}
bquery = escapeQuotationInSql(bquery);
- splitColumn = escapeQuotationInSql(dataSource.convertColumn(splitColumn, FlatTableSqlQuoteUtils.QUOTE));
+ splitColumn = escapeQuotationInSql(dataSource.convertColumn(splitColumn, FlatTableSqlQuoteUtils.getQuote()));
String cmd = StringUtils.format(
"--connect \"%s\" --driver %s --username %s --password %s --query \"%s AND \\$CONDITIONS\" "
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaInputBase.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaInputBase.java
index 23b0ae0..b159652 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaInputBase.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaInputBase.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.job.util.FlatTableSqlQuoteUtils;
import org.apache.kylin.source.kafka.model.StreamCubeFactTableDesc;
import org.apache.kylin.engine.mr.IInput;
import org.apache.kylin.engine.mr.JobBuilderSupport;
@@ -34,7 +35,6 @@ import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.util.FlatTableSqlQuoteUtils;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.source.hive.CreateFlatHiveTableStep;
import org.apache.kylin.source.hive.GarbageCollectionStep;
@@ -158,7 +158,7 @@ public class KafkaInputBase {
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, baseLocation);
String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
- insertDataHqls = insertDataHqls.replace(flatDesc.getDataModel().getRootFactTable().getTableIdentityQuoted(FlatTableSqlQuoteUtils.QUOTE) + " ",
+ insertDataHqls = insertDataHqls.replace(flatDesc.getDataModel().getRootFactTable().getTableIdentityQuoted(FlatTableSqlQuoteUtils.getQuote()) + " ",
quoteTableIdentity(hiveTableDatabase, streamFactDesc.getTableName()) + " ");
CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();