You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/08/24 09:33:54 UTC

[1/2] incubator-kylin git commit: KYLIN-900 Filter should to use table name and not generated alias

Repository: incubator-kylin
Updated Branches:
  refs/heads/0.7-staging 0aaa0f54b -> 572bcbc1b


KYLIN-900 Filter should to use table name and not generated alias


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/af7f9678
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/af7f9678
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/af7f9678

Branch: refs/heads/0.7-staging
Commit: af7f96787032ab37b4bd3688a0924c9382cabb81
Parents: 0aaa0f5
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Aug 24 15:32:43 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Aug 24 15:32:43 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/job/JoinedFlatTable.java   | 25 +++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/af7f9678/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 13e5e5f..03119c3 100644
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -52,10 +52,6 @@ import org.xml.sax.SAXException;
 
 public class JoinedFlatTable {
 
-    public static final String FACT_TABLE_ALIAS = "FACT_TABLE";
-
-    public static final String LOOKUP_TABLE_ALAIS_PREFIX = "LOOKUP_";
-
     public static String getTableDir(IJoinedFlatTableDesc intermediateTableDesc, String storageDfsDir, String jobUUID) {
         return storageDfsDir + "/" + intermediateTableDesc.getTableName(jobUUID);
     }
@@ -147,20 +143,31 @@ public class JoinedFlatTable {
     private static Map<String, String> buildTableAliasMap(DataModelDesc dataModelDesc) {
         Map<String, String> tableAliasMap = new HashMap<String, String>();
 
-        tableAliasMap.put(dataModelDesc.getFactTable().toUpperCase(), FACT_TABLE_ALIAS);
+        addTableAlias(dataModelDesc.getFactTable(), tableAliasMap);
 
-        int i = 1;
         for (LookupDesc lookupDesc : dataModelDesc.getLookups()) {
             JoinDesc join = lookupDesc.getJoin();
             if (join != null) {
-                tableAliasMap.put(lookupDesc.getTable().toUpperCase(), LOOKUP_TABLE_ALAIS_PREFIX + i);
-                i++;
+                addTableAlias(lookupDesc.getTable(), tableAliasMap);
             }
-
         }
         return tableAliasMap;
     }
 
+    // The table alias used to be "FACT_TABLE" and "LOOKUP_#", but that's too unpredictable
+    // for those who want to write a filter. (KYLIN-900)
+    // Also yet don't support joining the same table more than once, since table name is the map key.
+    private static void addTableAlias(String table, Map<String, String> tableAliasMap) {
+        String alias;
+        int cut = table.lastIndexOf('.');
+        if (cut < 0)
+            alias = table;
+        else
+            alias = table.substring(cut + 1);
+        
+        tableAliasMap.put(table, alias);
+    }
+
     private static void appendJoinStatement(IJoinedFlatTableDesc intermediateTableDesc, StringBuilder sql, Map<String, String> tableAliasMap) {
         Set<String> dimTableCache = new HashSet<String>();
 


[2/2] incubator-kylin git commit: minor, fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows

Posted by li...@apache.org.
minor, fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/572bcbc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/572bcbc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/572bcbc1

Branch: refs/heads/0.7-staging
Commit: 572bcbc1bf9fce7332124063fb119b8f9ac90ae6
Parents: af7f967
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Aug 24 15:33:29 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Aug 24 15:33:29 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/hadoop/AbstractHadoopJob.java     | 62 ++++++++++----------
 1 file changed, 32 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/572bcbc1/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index f426618..6ad89d6 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -234,35 +234,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
         }
 
         dumpResources(kylinConfig, metaDir, dumpList);
-
-        // hadoop distributed cache
-        String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
-        logger.info("HDFS meta dir is: " + hdfsMetaDir);
-        conf.set("tmpfiles", hdfsMetaDir);
-
-    }
-
-    protected void cleanupTempConfFile(Configuration conf) {
-        String tempMetaFileString = conf.get("tmpfiles");
-        logger.info("tempMetaFileString is : " + tempMetaFileString);
-        if (tempMetaFileString != null) {
-            if (tempMetaFileString.startsWith("file://")) {
-                tempMetaFileString = tempMetaFileString.substring("file://".length());
-                File tempMetaFile = new File(tempMetaFileString);
-                if (tempMetaFile.exists()) {
-                    try {
-                        FileUtils.forceDelete(tempMetaFile.getParentFile());
-
-                    } catch (IOException e) {
-                        logger.warn("error when deleting " + tempMetaFile, e);
-                    }
-                } else {
-                    logger.info("" + tempMetaFileString + " does not exist");
-                }
-            } else {
-                logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
-            }
-        }
+        addToHadoopDistCache(conf, metaDir);
     }
 
     protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
@@ -293,9 +265,16 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
         }
 
         dumpResources(kylinConfig, metaDir, dumpList);
+        addToHadoopDistCache(conf, metaDir);
+    }
 
+    private void addToHadoopDistCache(Configuration conf, File metaDir) {
         // hadoop distributed cache
-        String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+        String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+        if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..."
+            hdfsMetaDir = "file://" + hdfsMetaDir;
+        else
+            hdfsMetaDir = "file:///" + hdfsMetaDir;
         logger.info("HDFS meta dir is: " + hdfsMetaDir);
         conf.set("tmpfiles", hdfsMetaDir);
     }
@@ -356,6 +335,29 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
         return kylinConfig;
     }
 
+    protected void cleanupTempConfFile(Configuration conf) {
+        String tempMetaFileString = conf.get("tmpfiles");
+        logger.info("tempMetaFileString is : " + tempMetaFileString);
+        if (tempMetaFileString != null) {
+            if (tempMetaFileString.startsWith("file://")) {
+                tempMetaFileString = tempMetaFileString.substring("file://".length());
+                File tempMetaFile = new File(tempMetaFileString);
+                if (tempMetaFile.exists()) {
+                    try {
+                        FileUtils.forceDelete(tempMetaFile.getParentFile());
+
+                    } catch (IOException e) {
+                        logger.warn("error when deleting " + tempMetaFile, e);
+                    }
+                } else {
+                    logger.info("" + tempMetaFileString + " does not exist");
+                }
+            } else {
+                logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
+            }
+        }
+    }
+
     public void kill() throws JobException {
         if (job != null) {
             try {