You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/04 11:23:21 UTC
[1/4] incubator-kylin git commit: fixed hadoop/hive lib directory
detection to work with MapR paths (such as /opt/mapr/hive/hive-0.13/)
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging 4ab18c689 -> fc838d3e9
fixed hadoop/hive lib directory detection to work with MapR paths (such as /opt/mapr/hive/hive-0.13/)
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bc8d3601
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bc8d3601
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bc8d3601
Branch: refs/heads/0.7-staging
Commit: bc8d36011c413a232415fbc436d2953290982d5e
Parents: 4ab18c6
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:30:53 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:36 2015 +0800
----------------------------------------------------------------------
bin/find-hive-dependency.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc8d3601/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index 817697f..d8accfc 100644
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -30,16 +30,16 @@ do
then
hive_exec_path=$data
fi
- result=`echo $data | grep 'hive/conf'`
+ result=`echo $data | grep -e 'hive[^/]*/conf'`
if [ $result ]
then
hive_conf_path=$data
fi
done
-hdp_home=`echo $hive_exec_path | awk -F '/hive/lib/' '{print $1}'`
+hdp_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
hcatalog=`find $hdp_home -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
-hive_lib=`find "$hdp_home/hive/lib" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
if [ -z "$hcatalog" ]
then
@@ -49,4 +49,4 @@ fi
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
echo "hive dependency: $hive_dependency"
-export hive_dependency
\ No newline at end of file
+export hive_dependency
[3/4] incubator-kylin git commit: added an exists check before
setting column family file permissions
Posted by li...@apache.org.
added an exists check before setting column family file permissions
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/02562c0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/02562c0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/02562c0d
Branch: refs/heads/0.7-staging
Commit: 02562c0dd7c4010726e3d3ada42c49cb279bd68a
Parents: 7427f74
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:46:58 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:44 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02562c0d/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
index fa3725c..7a4702e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
@@ -72,7 +72,12 @@ public class BulkLoadJob extends AbstractHadoopJob {
FsPermission permission = new FsPermission((short) 0777);
for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
String cfName = cf.getName();
- fs.setPermission(new Path(input + cfName), permission);
+ Path columnFamilyPath = new Path(input + cfName);
+
+ // File may have already been auto-loaded (in the case of MapR DB)
+ if(fs.exists(columnFamilyPath)) {
+ fs.setPermission(columnFamilyPath, permission);
+ }
}
String[] newArgs = new String[2];
[4/4] incubator-kylin git commit: added accepts json request header
so RM responds with JSON rather than XML
Posted by li...@apache.org.
added accepts json request header so RM responds with JSON rather than XML
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fc838d3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fc838d3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fc838d3e
Branch: refs/heads/0.7-staging
Commit: fc838d3e919d1224ba1a93897ae3440b3abaa79a
Parents: 02562c0
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:48:24 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:48 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc838d3e/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 2a9abf9..b9ceee0 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -72,6 +72,8 @@ public class HadoopStatusGetter {
}
HttpMethod get = new GetMethod(url);
+ get.addRequestHeader("accept", "application/json");
+
try {
client.executeMethod(get);
[2/4] incubator-kylin git commit: altered command line pattern
recognition to work with MapRFS
Posted by li...@apache.org.
altered command line pattern recognition to work with MapRFS
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7427f740
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7427f740
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7427f740
Branch: refs/heads/0.7-staging
Commit: 7427f740e2157e6a6768c6a65f13a0ed3c7d8bc4
Parents: bc8d360
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:35:24 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:40 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/job/common/ShellExecutable.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7427f740/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 47a3145..4bc8dac 100644
--- a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -73,13 +73,13 @@ public class ShellExecutable extends AbstractExecutable {
private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager");
private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)");
private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)");
- private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("HDFS: Number of bytes written=(\\d+)");
+ private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)");
- private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("HDFS Read: (\\d+) HDFS Write");
+ private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write");
// hive
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
- private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("HDFS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
@Override
public void log(String message) {