You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by GitBox <gi...@apache.org> on 2019/06/25 12:56:22 UTC
[GitHub] [kylin] nichunen commented on a change in pull request #710:
KYLIN-4046 Refine JDBC Source(source.default=8)
nichunen commented on a change in pull request #710: KYLIN-4046 Refine JDBC Source(source.default=8)
URL: https://github.com/apache/kylin/pull/710#discussion_r297172308
##########
File path: source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
##########
@@ -229,4 +298,245 @@ protected static String escapeQuotationInSql(String sqlExpr) {
sqlExpr = sqlExpr.replaceAll("`", "\\\\`");
return sqlExpr;
}
+
+ private static String generateSelectDataStatementRDBMS(IJoinedFlatTableDesc flatDesc, boolean singleLine,
+ String[] skipAs, IJdbcMetadata metadata, Map<String, String> metaMap) {
+ SourceDialect dialect = metadata.getDialect();
+ final String sep = singleLine ? " " : "\n";
+
+ final List<String> skipAsList = (skipAs == null) ? new ArrayList<>() : Arrays.asList(skipAs);
+
+ StringBuilder sql = new StringBuilder();
+ sql.append("SELECT");
+ sql.append(sep);
+
+ for (int i = 0; i < flatDesc.getAllColumns().size(); i++) {
+ TblColRef col = flatDesc.getAllColumns().get(i);
+ if (i > 0) {
+ sql.append(",");
+ }
+ String colTotalName = String.format(Locale.ROOT, "%s.%s", col.getTableRef().getTableName(), col.getName());
+ if (skipAsList.contains(colTotalName)) {
+ sql.append(getTableColumnIdentityQuoted(col, metadata, metaMap, true)).append(sep);
+ } else {
+ sql.append(getTableColumnIdentityQuoted(col, metadata, metaMap, true)).append(" as ")
+ .append(quoteIdentifier(JoinedFlatTable.colName(col), dialect)).append(sep);
+ }
+ }
+ appendJoinStatement(flatDesc, sql, singleLine, metadata, metaMap);
+ appendWhereStatement(flatDesc, sql, singleLine, metadata, metaMap);
+ return sql.toString();
+ }
+
+ private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
+ IJdbcMetadata metadata, Map<String, String> metaMap) {
+ SourceDialect dialect = metadata.getDialect();
+ final String sep = singleLine ? " " : "\n";
+ Set<TableRef> dimTableCache = new HashSet<>();
+
+ DataModelDesc model = flatDesc.getDataModel();
+ sql.append(" FROM ")
+ .append(getSchemaQuoted(metaMap,
+ flatDesc.getDataModel().getRootFactTable().getTableDesc().getDatabase(), metadata, true))
+ .append(".")
+ .append(getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true));
+
+ sql.append(" ");
+ sql.append((getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true)))
+ .append(sep);
+
+ for (JoinTableDesc lookupDesc : model.getJoinTables()) {
+ JoinDesc join = lookupDesc.getJoin();
+ if (join != null && !join.getType().equals("")) {
+ TableRef dimTable = lookupDesc.getTableRef();
+ if (!dimTableCache.contains(dimTable)) {
+ TblColRef[] pk = join.getPrimaryKeyColumns();
+ TblColRef[] fk = join.getForeignKeyColumns();
+ if (pk.length != fk.length) {
+ throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
+ }
+ String joinType = join.getType().toUpperCase(Locale.ROOT);
+
+ sql.append(joinType).append(" JOIN ")
+ .append(getSchemaQuoted(metaMap, dimTable.getTableDesc().getDatabase(), metadata, true))
+ .append(".").append(getTableIdentityQuoted(dimTable, metaMap, metadata, true));
+
+ sql.append(" ");
+ sql.append(getTableIdentityQuoted(dimTable, metaMap, metadata, true)).append(sep);
+ sql.append("ON ");
+ for (int i = 0; i < pk.length; i++) {
+ if (i > 0) {
+ sql.append(" AND ");
+ }
+ sql.append(getTableColumnIdentityQuoted(fk[i], metadata, metaMap, true)).append(" = ")
+ .append(getTableColumnIdentityQuoted(pk[i], metadata, metaMap, true));
+ }
+ sql.append(sep);
+ dimTableCache.add(dimTable);
+ }
+ }
+ }
+ }
+
+ private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
+ IJdbcMetadata metadata, Map<String, String> metaMap) {
+ final String sep = singleLine ? " " : "\n";
+
+ StringBuilder whereBuilder = new StringBuilder();
+ whereBuilder.append("WHERE 1=1");
+
+ DataModelDesc model = flatDesc.getDataModel();
+ if (StringUtils.isNotEmpty(model.getFilterCondition())) {
+ whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
+ }
+
+ if (flatDesc.getSegment() != null) {
+ PartitionDesc partDesc = model.getPartitionDesc();
+ if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
+ SegmentRange segRange = flatDesc.getSegRange();
+
+ if (segRange != null && !segRange.isInfinite()) {
+ whereBuilder.append(" AND (");
+ whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
+ flatDesc.getSegment(), segRange,
+ col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
+ whereBuilder.append(")");
+ whereBuilder.append(sep);
+ }
+ }
+ }
+ sql.append(whereBuilder.toString());
+ }
+
+ /**
+ * @return {TABLE_NAME}.{COLUMN_NAME}
+ */
+ private static String getTableColumnIdentityQuoted(TblColRef col, IJdbcMetadata metadata,
+ Map<String, String> metaMap, boolean needQuote) {
+ String tblName = getTableIdentityQuoted(col.getTableRef(), metaMap, metadata, false);
+ String colName = getColumnIdentityQuoted(col, metadata, metaMap, needQuote);
+ if (needQuote)
+ tblName = quoteIdentifier(tblName, metadata.getDialect());
+ return tblName + "." + colName;
+ }
+
+ /**
+ * @return {SCHEMA_NAME}
+ */
+ static String getSchemaQuoted(Map<String, String> metaMap, String database, IJdbcMetadata metadata,
+ boolean needQuote) {
+ metaMap.get(database.toUpperCase(Locale.ROOT));
Review comment:
what's this for?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services