You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/08/24 09:33:54 UTC
[1/2] incubator-kylin git commit: KYLIN-900 Filter should to use
table name and not generated alias
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging 0aaa0f54b -> 572bcbc1b
KYLIN-900 Filter should to use table name and not generated alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/af7f9678
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/af7f9678
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/af7f9678
Branch: refs/heads/0.7-staging
Commit: af7f96787032ab37b4bd3688a0924c9382cabb81
Parents: 0aaa0f5
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Aug 24 15:32:43 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Aug 24 15:32:43 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/JoinedFlatTable.java | 25 +++++++++++++-------
1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af7f9678/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 13e5e5f..03119c3 100644
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -52,10 +52,6 @@ import org.xml.sax.SAXException;
public class JoinedFlatTable {
- public static final String FACT_TABLE_ALIAS = "FACT_TABLE";
-
- public static final String LOOKUP_TABLE_ALAIS_PREFIX = "LOOKUP_";
-
public static String getTableDir(IJoinedFlatTableDesc intermediateTableDesc, String storageDfsDir, String jobUUID) {
return storageDfsDir + "/" + intermediateTableDesc.getTableName(jobUUID);
}
@@ -147,20 +143,31 @@ public class JoinedFlatTable {
private static Map<String, String> buildTableAliasMap(DataModelDesc dataModelDesc) {
Map<String, String> tableAliasMap = new HashMap<String, String>();
- tableAliasMap.put(dataModelDesc.getFactTable().toUpperCase(), FACT_TABLE_ALIAS);
+ addTableAlias(dataModelDesc.getFactTable(), tableAliasMap);
- int i = 1;
for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
JoinDesc join = lookupDesc.getJoin();
if (join != null) {
- tableAliasMap.put(lookupDesc.getTable().toUpperCase(), LOOKUP_TABLE_ALAIS_PREFIX + i);
- i++;
+ addTableAlias(lookupDesc.getTable(), tableAliasMap);
}
-
}
return tableAliasMap;
}
+ // The table alias used to be "FACT_TABLE" and "LOOKUP_#", but that's too unpredictable
+ // for those who want to write a filter. (KYLIN-900)
+ // Also yet don't support joining the same table more than once, since table name is the map key.
+ private static void addTableAlias(String table, Map<String, String> tableAliasMap) {
+ String alias;
+ int cut = table.lastIndexOf('.');
+ if (cut < 0)
+ alias = table;
+ else
+ alias = table.substring(cut + 1);
+
+ tableAliasMap.put(table, alias);
+ }
+
private static void appendJoinStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
Set<String> dimTableCache = new HashSet<String>();
[2/2] incubator-kylin git commit: minor,
fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows
Posted by li...@apache.org.
minor, fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/572bcbc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/572bcbc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/572bcbc1
Branch: refs/heads/0.7-staging
Commit: 572bcbc1bf9fce7332124063fb119b8f9ac90ae6
Parents: af7f967
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Aug 24 15:33:29 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Aug 24 15:33:29 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/AbstractHadoopJob.java | 62 ++++++++++----------
1 file changed, 32 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/572bcbc1/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index f426618..6ad89d6 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -234,35 +234,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
dumpResources(kylinConfig, metaDir, dumpList);
-
- // hadoop distributed cache
- String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
- logger.info("HDFS meta dir is: " + hdfsMetaDir);
- conf.set("tmpfiles", hdfsMetaDir);
-
- }
-
- protected void cleanupTempConfFile(Configuration conf) {
- String tempMetaFileString = conf.get("tmpfiles");
- logger.info("tempMetaFileString is : " + tempMetaFileString);
- if (tempMetaFileString != null) {
- if (tempMetaFileString.startsWith("file://")) {
- tempMetaFileString = tempMetaFileString.substring("file://".length());
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- try {
- FileUtils.forceDelete(tempMetaFile.getParentFile());
-
- } catch (IOException e) {
- logger.warn("error when deleting " + tempMetaFile, e);
- }
- } else {
- logger.info("" + tempMetaFileString + " does not exist");
- }
- } else {
- logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
- }
- }
+ addToHadoopDistCache(conf, metaDir);
}
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
@@ -293,9 +265,16 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
dumpResources(kylinConfig, metaDir, dumpList);
+ addToHadoopDistCache(conf, metaDir);
+ }
+ private void addToHadoopDistCache(Configuration conf, File metaDir) {
// hadoop distributed cache
- String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..."
+ hdfsMetaDir = "file://" + hdfsMetaDir;
+ else
+ hdfsMetaDir = "file:///" + hdfsMetaDir;
logger.info("HDFS meta dir is: " + hdfsMetaDir);
conf.set("tmpfiles", hdfsMetaDir);
}
@@ -356,6 +335,29 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return kylinConfig;
}
+ protected void cleanupTempConfFile(Configuration conf) {
+ String tempMetaFileString = conf.get("tmpfiles");
+ logger.info("tempMetaFileString is : " + tempMetaFileString);
+ if (tempMetaFileString != null) {
+ if (tempMetaFileString.startsWith("file://")) {
+ tempMetaFileString = tempMetaFileString.substring("file://".length());
+ File tempMetaFile = new File(tempMetaFileString);
+ if (tempMetaFile.exists()) {
+ try {
+ FileUtils.forceDelete(tempMetaFile.getParentFile());
+
+ } catch (IOException e) {
+ logger.warn("error when deleting " + tempMetaFile, e);
+ }
+ } else {
+ logger.info("" + tempMetaFileString + " does not exist");
+ }
+ } else {
+ logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
+ }
+ }
+ }
+
public void kill() throws JobException {
if (job != null) {
try {