You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/06/08 20:22:24 UTC
[01/11] incubator-kylin git commit: KYLIN-758 For HortonWorks Sandbox
2.2, these two ports are not forwad.. (Rebased from
https://github.com/apache/incubator-kylin/pull/4,
contributed by SleepyThread & akashm)
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7 48823ca83 -> 724ccf96d
KYLIN-758 For HortonWorks Sandbox 2.2, these two ports are not forwad.. (Rebased from https://github.com/apache/incubator-kylin/pull/4, contributed by SleepyThread & akashm)
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4ab18c68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4ab18c68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4ab18c68
Branch: refs/heads/0.7
Commit: 4ab18c689726634ec2efa20214804951b89ec5f8
Parents: f5dc6b9
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jun 4 16:19:35 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 16:19:35 2015 +0800
----------------------------------------------------------------------
docs/Installation/Hadoop Environment.md | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4ab18c68/docs/Installation/Hadoop Environment.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Hadoop Environment.md b/docs/Installation/Hadoop Environment.md
index 2e72527..5ee12fe 100644
--- a/docs/Installation/Hadoop Environment.md
+++ b/docs/Installation/Hadoop Environment.md
@@ -18,4 +18,9 @@ Use ambari helps to launch hadoop:
With both command successfully run you can go to ambari homepage at <http://your_sandbox_ip:8080> (user:admin,password:admin) to check everything's status. **By default hortonworks ambari disables Hbase, you'll need manually start the `Hbase` service at ambari homepage.**
-
\ No newline at end of file
+
+
+**Additonal Info for setting up HortonWorks Sandbox on Virtual Box**
+
+ Please make sure Hbase Master port [Default 60000] and Zookeeper [Default 2181] is forwarded to Host OS.
+
\ No newline at end of file
[05/11] incubator-kylin git commit: added accepts json request header
so RM responds with JSON rather than XML
Posted by lu...@apache.org.
added accepts json request header so RM responds with JSON rather than XML
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fc838d3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fc838d3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fc838d3e
Branch: refs/heads/0.7
Commit: fc838d3e919d1224ba1a93897ae3440b3abaa79a
Parents: 02562c0
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:48:24 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:48 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fc838d3e/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 2a9abf9..b9ceee0 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -72,6 +72,8 @@ public class HadoopStatusGetter {
}
HttpMethod get = new GetMethod(url);
+ get.addRequestHeader("accept", "application/json");
+
try {
client.executeMethod(get);
[08/11] incubator-kylin git commit: KYLIN-815
IPartitionConditionBuilder, flexible range condition to match hive partitions
Posted by lu...@apache.org.
KYLIN-815 IPartitionConditionBuilder, flexible range condition to match hive partitions
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1dc1a370
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1dc1a370
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1dc1a370
Branch: refs/heads/0.7
Commit: 1dc1a370dc0707925757d7dff4e2f05bf65b3be5
Parents: d2ae503
Author: Yang Li <li...@apache.org>
Authored: Sun Jun 7 16:35:34 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Jun 7 16:35:34 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ClassUtil.java | 7 ++
.../apache/kylin/common/util/DateFormat.java | 101 +++++++++++++++++++
.../apache/kylin/dict/DateStrDictionary.java | 66 +-----------
.../kylin/invertedindex/index/TableRecord.java | 9 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 50 +++------
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 12 ++-
.../kylin/metadata/model/PartitionDesc.java | 79 +++++++++++----
.../kylin/storage/hbase/HBaseKeyRange.java | 14 +--
.../org/apache/kylin/storage/tuple/Tuple.java | 6 +-
9 files changed, 209 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index b474c58..4703e86 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -50,4 +50,11 @@ public class ClassUtil {
return (Class<? extends T>) Class.forName(name);
}
+ public static Object newInstance(String clz) {
+ try {
+ return forName(clz, Object.class).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
new file mode 100644
index 0000000..7aca923
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common.util;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class DateFormat {
+
+ public static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd";
+ public static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
+ public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
+
+ static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
+
+ public static SimpleDateFormat getDateFormat(String datePattern) {
+ ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
+ if (formatThreadLocal == null) {
+ threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
+ }
+ SimpleDateFormat format = formatThreadLocal.get();
+ if (format == null) {
+ format = new SimpleDateFormat(datePattern);
+ format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
+ formatThreadLocal.set(format);
+ }
+ return format;
+ }
+
+ public static String formatToDateStr(long millis) {
+ return getDateFormat(DEFAULT_DATE_PATTERN).format(new Date(millis));
+ }
+
+ public static String formatToTimeStr(long millis) {
+ return getDateFormat(DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).format(new Date(millis));
+ }
+
+ public static String dateToString(Date date) {
+ return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ }
+
+ public static String dateToString(Date date, String pattern) {
+ return getDateFormat(pattern).format(date);
+ }
+
+ public static Date stringToDate(String str) {
+ return stringToDate(str, DEFAULT_DATE_PATTERN);
+ }
+
+ public static Date stringToDate(String str, String pattern) {
+ Date date = null;
+ try {
+ date = getDateFormat(pattern).parse(str);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e);
+ }
+ return date;
+ }
+
+ public static long stringToMillis(String str) {
+ if (isAllDigits(str)) {
+ return Long.parseLong(str);
+ } else if (str.length() == 10) {
+ return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
+ } else if (str.length() == 19) {
+ return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
+ } else if (str.length() == 23) {
+ return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
+ } else {
+ throw new IllegalArgumentException("there is no valid date pattern for:" + str);
+ }
+ }
+
+ private static boolean isAllDigits(String str) {
+ for (int i = 0, n = str.length(); i < n; i++) {
+ if (Character.isDigit(str.charAt(i)) == false)
+ return false;
+ }
+ return true;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 7cace15..ab4915a 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -23,14 +23,10 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.Date;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
/**
* A dictionary for date string (date only, no time).
@@ -44,62 +40,8 @@ import org.apache.commons.lang.StringUtils;
*/
public class DateStrDictionary extends Dictionary<String> {
- static final String DEFAULT_DATE_PATTERN = "yyyy-MM-dd";
- static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
- static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
-
static final int ID_9999_12_31 = 3652426; // assume 0 based
- static final private Map<String, ThreadLocal<SimpleDateFormat>> threadLocalMap = new ConcurrentHashMap<String, ThreadLocal<SimpleDateFormat>>();
-
- static SimpleDateFormat getDateFormat(String datePattern) {
- ThreadLocal<SimpleDateFormat> formatThreadLocal = threadLocalMap.get(datePattern);
- if (formatThreadLocal == null) {
- threadLocalMap.put(datePattern, formatThreadLocal = new ThreadLocal<SimpleDateFormat>());
- }
- SimpleDateFormat format = formatThreadLocal.get();
- if (format == null) {
- format = new SimpleDateFormat(datePattern);
- format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
- formatThreadLocal.set(format);
- }
- return format;
- }
-
- public static String dateToString(Date date) {
- return dateToString(date, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- }
-
- public static String dateToString(Date date, String pattern) {
- return getDateFormat(pattern).format(date);
- }
-
- public static Date stringToDate(String str) {
- return stringToDate(str, DEFAULT_DATE_PATTERN);
- }
-
- public static Date stringToDate(String str, String pattern) {
- Date date = null;
- try {
- date = getDateFormat(pattern).parse(str);
- } catch (ParseException e) {
- throw new IllegalArgumentException("'" + str + "' is not a valid date of pattern '" + pattern + "'", e);
- }
- return date;
- }
-
- public static long stringToMillis(String str) {
- if (str.length() == 10) {
- return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
- } else if (str.length() == 19) {
- return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
- } else if (str.length() == 23) {
- return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS).getTime();
- } else {
- throw new IllegalArgumentException("there is no valid date pattern for:" + str);
- }
- }
-
// ============================================================================
private String pattern;
@@ -107,7 +49,7 @@ public class DateStrDictionary extends Dictionary<String> {
private int maxId;
public DateStrDictionary() {
- init(DEFAULT_DATE_PATTERN, 0);
+ init(DateFormat.DEFAULT_DATE_PATTERN, 0);
}
public DateStrDictionary(String datePattern, int baseId) {
@@ -147,7 +89,7 @@ public class DateStrDictionary extends Dictionary<String> {
@Override
final protected int getIdFromValueImpl(String value, int roundFlag) {
- Date date = stringToDate(value, pattern);
+ Date date = DateFormat.stringToDate(value, pattern);
int id = calcIdFromSeqNo(getNumOfDaysSince0000(date));
if (id < baseId || id > maxId)
throw new IllegalArgumentException("'" + value + "' encodes to '" + id + "' which is out of range [" + baseId + "," + maxId + "]");
@@ -160,7 +102,7 @@ public class DateStrDictionary extends Dictionary<String> {
if (id < baseId || id > maxId)
throw new IllegalArgumentException("ID '" + id + "' is out of range [" + baseId + "," + maxId + "]");
Date d = getDateFromNumOfDaysSince0000(calcSeqNoFromId(id));
- return dateToString(d, pattern);
+ return DateFormat.dateToString(d, pattern);
}
private int getNumOfDaysSince0000(Date d) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
index 3b8d969..d10554b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
@@ -18,13 +18,14 @@
package org.apache.kylin.invertedindex.index;
-import com.google.common.collect.Lists;
-import org.apache.kylin.dict.DateStrDictionary;
+import java.util.List;
+
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.util.DateFormat;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* @author yangli9, honma
@@ -66,7 +67,7 @@ public class TableRecord implements Cloneable {
public long getTimestamp() {
String str = getValueString(info.getTimestampColumn());
- return DateStrDictionary.stringToMillis(str);
+ return DateFormat.stringToMillis(str);
}
public int length(int col) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7e714d0..db6dd0e 100644
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -20,19 +20,15 @@ package org.apache.kylin.job;
import java.io.File;
import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -43,7 +39,11 @@ import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
/**
* @author George Song (ysong1)
@@ -144,7 +144,7 @@ public class JoinedFlatTable {
return sql.toString();
}
- private static Map buildTableAliasMap(DataModelDesc dataModelDesc) {
+ private static Map<String, String> buildTableAliasMap(DataModelDesc dataModelDesc) {
Map<String, String> tableAliasMap = new HashMap<String, String>();
tableAliasMap.put(dataModelDesc.getFactTable().toUpperCase(), FACT_TABLE_ALIAS);
@@ -207,36 +207,23 @@ public class JoinedFlatTable {
whereBuilder.append("WHERE");
CubeDesc cubeDesc = desc.getCubeDesc();
+ DataModelDesc model = cubeDesc.getModel();
- if (cubeDesc.getModel().getFilterCondition() != null && cubeDesc.getModel().getFilterCondition().equals("") == false) {
- whereBuilder.append(" (").append(cubeDesc.getModel().getFilterCondition()).append(") ");
+ if (model.getFilterCondition() != null && model.getFilterCondition().equals("") == false) {
+ whereBuilder.append(" (").append(model.getFilterCondition()).append(") ");
hasCondition = true;
}
CubeSegment cubeSegment = desc.getCubeSegment();
if (null != cubeSegment) {
+ PartitionDesc partDesc = model.getPartitionDesc();
long dateStart = cubeSegment.getDateRangeStart();
long dateEnd = cubeSegment.getDateRangeEnd();
if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) {
- String partitionColumnName = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumn();
- int indexOfDot = partitionColumnName.lastIndexOf(".");
-
- // convert to use table alias;
- if (indexOfDot > 0) {
- String partitionTableName = partitionColumnName.substring(0, indexOfDot);
- String columeOnly = partitionColumnName.substring(indexOfDot);
- String partitionTableAlias = tableAliasMap.get(partitionTableName);
- partitionColumnName = partitionTableAlias + columeOnly;
- }
-
whereBuilder.append(hasCondition ? " AND (" : " (");
- if (dateStart > 0) {
- whereBuilder.append(partitionColumnName + " >= '" + formatDateTimeInWhereClause(dateStart) + "' ");
- whereBuilder.append("AND ");
- }
- whereBuilder.append(partitionColumnName + " < '" + formatDateTimeInWhereClause(dateEnd) + "'");
+ whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd, tableAliasMap));
whereBuilder.append(")\n");
hasCondition = true;
}
@@ -247,15 +234,6 @@ public class JoinedFlatTable {
}
}
- private static String formatDateTimeInWhereClause(long datetime) {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- Date date = new Date(datetime);
- String str = f.format(date);
- // note "2014-10-01" >= "2014-10-01 00:00:00" is FALSE
- return StringUtil.dropSuffix(str, " 00:00:00");
- }
-
private static String colName(String canonicalColName) {
return canonicalColName.replace(".", "_");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
index a972692..ae66a18 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hive/CubeJoinedFlatTableDesc.java
@@ -18,18 +18,20 @@
package org.apache.kylin.job.hadoop.hive;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesSplitter;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* @author George Song (ysong1)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index cf0a8e3..73db41f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -20,15 +20,16 @@ package org.apache.kylin.metadata.model;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.StringSplitter;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.util.StringSplitter;
/**
- * @author xduo
- *
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class PartitionDesc {
@@ -44,31 +45,37 @@ public class PartitionDesc {
private long partitionDateStart = 0L;
@JsonProperty("partition_type")
private PartitionType partitionType = PartitionType.APPEND;
+ @JsonProperty("partition_condition_builder")
+ private String partitionConditionBuilderClz = DefaultPartitionConditionBuilder.class.getName();
private TblColRef partitionDateColumnRef;
+ private IPartitionConditionBuilder partitionConditionBuilder;
public void init(Map<String, TableDesc> tables) {
- if (StringUtils.isNotEmpty(partitionDateColumn)) {
- partitionDateColumn = partitionDateColumn.toUpperCase();
+ if (StringUtils.isEmpty(partitionDateColumn))
+ return;
- String[] columns = StringSplitter.split(partitionDateColumn, ".");
+ partitionDateColumn = partitionDateColumn.toUpperCase();
- if (null != columns && columns.length == 3) {
- String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
+ String[] columns = StringSplitter.split(partitionDateColumn, ".");
- TableDesc table = tables.get(tableName);
- ColumnDesc col = table.findColumnByName(columns[2]);
- if (col != null) {
- partitionDateColumnRef = new TblColRef(col);
- } else {
- throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
- }
+ if (null != columns && columns.length == 3) {
+ String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
+
+ TableDesc table = tables.get(tableName);
+ ColumnDesc col = table.findColumnByName(columns[2]);
+ if (col != null) {
+ partitionDateColumnRef = new TblColRef(col);
} else {
- throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
+ throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
}
+ } else {
+ throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
}
+
+ partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
-
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -93,6 +100,10 @@ public class PartitionDesc {
return partitionType;
}
+ public IPartitionConditionBuilder getPartitionConditionBuilder() {
+ return partitionConditionBuilder;
+ }
+
public void setCubePartitionType(PartitionType partitionType) {
this.partitionType = partitionType;
}
@@ -101,4 +112,36 @@ public class PartitionDesc {
return partitionDateColumnRef;
}
+ // ============================================================================
+
+ public static interface IPartitionConditionBuilder {
+ String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias);
+ }
+
+ public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder {
+
+ @Override
+ public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
+ String partitionColumnName = partDesc.getPartitionDateColumn();
+
+ // convert to use table alias
+ int indexOfDot = partitionColumnName.lastIndexOf(".");
+ if (indexOfDot > 0) {
+ String partitionTableName = partitionColumnName.substring(0, indexOfDot);
+ if (tableAlias != null && tableAlias.containsKey(partitionTableName))
+ partitionColumnName = tableAlias.get(partitionTableName) + partitionColumnName.substring(indexOfDot);
+ }
+
+ StringBuilder builder = new StringBuilder();
+
+ if (startInclusive > 0) {
+ builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive) + "' ");
+ builder.append("AND ");
+ }
+ builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive) + "'");
+
+ return builder.toString();
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
index 7926b1d..cb2dce7 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseKeyRange.java
@@ -25,11 +25,9 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Pair;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
@@ -37,9 +35,11 @@ import org.apache.kylin.cube.kv.FuzzyKeyEncoder;
import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.DateStrDictionary;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
/**
*
* @author xjiang
@@ -145,10 +145,10 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
private void initPartitionRange(ColumnValueRange dimRange) {
if (null != dimRange.getBeginValue()) {
- this.partitionColumnStartDate = DateStrDictionary.stringToDate(dimRange.getBeginValue()).getTime();
+ this.partitionColumnStartDate = DateFormat.stringToDate(dimRange.getBeginValue()).getTime();
}
if (null != dimRange.getEndValue()) {
- this.partitionColumnEndDate = DateStrDictionary.stringToDate(dimRange.getEndValue()).getTime();
+ this.partitionColumnEndDate = DateFormat.stringToDate(dimRange.getEndValue()).getTime();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1dc1a370/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 96bdaa6..5023580 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -23,10 +23,10 @@ import java.util.Date;
import java.util.List;
import org.apache.kylin.common.util.Array;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
-import org.apache.kylin.dict.DateStrDictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
@@ -133,7 +133,7 @@ public class Tuple implements ITuple {
// TODO use data type enum instead of string comparison
if ("date".equals(dataType)) {
// convert epoch time
- Date dateValue = DateStrDictionary.stringToDate(strValue); // NOTE: forces GMT timezone
+ Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
long millis = dateValue.getTime();
long days = millis / (1000 * 3600 * 24);
return Integer.valueOf((int) days); // Optiq expects Integer instead of Long. by honma
@@ -150,7 +150,7 @@ public class Tuple implements ITuple {
} else if ("decimal".equals(dataType)) {
return new BigDecimal(strValue);
} else if ("timestamp".equals(dataType)) {
- return Long.valueOf(DateStrDictionary.stringToMillis(strValue));
+ return Long.valueOf(DateFormat.stringToMillis(strValue));
} else if ("float".equals(dataType)){
return Float.valueOf(strValue);
} else {
[02/11] incubator-kylin git commit: fixed hadoop/hive lib directory
detection to work with MapR paths (such as /opt/mapr/hive/hive-0.13/)
Posted by lu...@apache.org.
fixed hadoop/hive lib directory detection to work with MapR paths (such as /opt/mapr/hive/hive-0.13/)
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bc8d3601
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bc8d3601
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bc8d3601
Branch: refs/heads/0.7
Commit: bc8d36011c413a232415fbc436d2953290982d5e
Parents: 4ab18c6
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:30:53 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:36 2015 +0800
----------------------------------------------------------------------
bin/find-hive-dependency.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bc8d3601/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index 817697f..d8accfc 100644
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -30,16 +30,16 @@ do
then
hive_exec_path=$data
fi
- result=`echo $data | grep 'hive/conf'`
+ result=`echo $data | grep -e 'hive[^/]*/conf'`
if [ $result ]
then
hive_conf_path=$data
fi
done
-hdp_home=`echo $hive_exec_path | awk -F '/hive/lib/' '{print $1}'`
+hdp_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
hcatalog=`find $hdp_home -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
-hive_lib=`find "$hdp_home/hive/lib" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
if [ -z "$hcatalog" ]
then
@@ -49,4 +49,4 @@ fi
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
echo "hive dependency: $hive_dependency"
-export hive_dependency
\ No newline at end of file
+export hive_dependency
[10/11] incubator-kylin git commit: resize select box in query page
and make grid selectable
Posted by lu...@apache.org.
resize select box in query page and make grid selectable
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d5a0e495
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d5a0e495
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d5a0e495
Branch: refs/heads/0.7
Commit: d5a0e495e691fb1dfff516084a3bd7c92d36d5cb
Parents: a7c28c3
Author: jiazhong <ji...@ebay.com>
Authored: Mon Jun 8 20:18:27 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Jun 8 20:18:27 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/query/query_detail.html | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d5a0e495/webapp/app/partials/query/query_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query_detail.html b/webapp/app/partials/query/query_detail.html
index 5ca9010..42f8dbb 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -128,7 +128,7 @@
</div>
<div ng-if="curQuery.graph.show" class="row">
- <div class="col-xs-4 col-md-3">
+ <div class="col-xs-4 col-lg-3">
<div class="graph_content">
<label>Graph Type</label>
@@ -174,7 +174,7 @@
</div>
</div>
</div>
- <div class="col-xs-8 col-md-9">
+ <div class="col-xs-8 col-lg-9">
<div ng-if="curQuery.graph.data.length > 0">
<div ng-if="curQuery.graph.type.value == 'bar'">
<nvd3-multi-bar-chart
[11/11] incubator-kylin git commit: Merge branch '0.7.1' into
0.7-staging
Posted by lu...@apache.org.
Merge branch '0.7.1' into 0.7-staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/724ccf96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/724ccf96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/724ccf96
Branch: refs/heads/0.7
Commit: 724ccf96dd75b649d0e0fa9daf93b8410f818565
Parents: d5a0e49 48823ca
Author: Luke Han <lu...@apache.org>
Authored: Mon Jun 8 10:12:23 2015 -0700
Committer: Luke Han <lu...@apache.org>
Committed: Mon Jun 8 10:12:23 2015 -0700
----------------------------------------------------------------------
atopcalcite/pom.xml | 2 +-
bin/find-hbase-dependency.sh | 18 +
common/pom.xml | 2 +-
cube/pom.xml | 2 +-
dictionary/pom.xml | 2 +-
.../apache/kylin/dict/TrieDictionaryTest.java | 2 +-
.../english-words.80 (scowl-2015.05.18).dic | 139304 ++++++++++++++++
.../english-words.80 (scowl-2015.05.18).txt | 139304 ----------------
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
job/pom.xml | 2 +-
metadata/pom.xml | 2 +-
pom.xml | 9 +-
query/pom.xml | 2 +-
server/pom.xml | 2 +-
src/main/config/assemblies/source-assembly.xml | 3 +
storage/pom.xml | 2 +-
17 files changed, 139343 insertions(+), 139319 deletions(-)
----------------------------------------------------------------------
[06/11] incubator-kylin git commit: minor,
DebugTomcat pick up job jars from dev env
Posted by lu...@apache.org.
minor, DebugTomcat pick up job jars from dev env
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/14141e0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/14141e0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/14141e0d
Branch: refs/heads/0.7
Commit: 14141e0dbc6bdc597c39628a548059e259447881
Parents: fc838d3
Author: Yang Li <li...@apache.org>
Authored: Sat Jun 6 20:52:20 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jun 6 20:52:20 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 3 +-
.../java/org/apache/kylin/rest/DebugTomcat.java | 87 ++++++++++++++------
2 files changed, 62 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/14141e0d/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 536f34c..b21e63b 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -304,6 +304,7 @@ public class KylinConfig {
}
public void overrideKylinJobJarPath(String path) {
+ logger.info("override " + KYLIN_JOB_JAR + " to " + path);
System.setProperty(KYLIN_JOB_JAR, path);
}
@@ -341,6 +342,7 @@ public class KylinConfig {
}
public void overrideCoprocessorLocalJar(String path) {
+ logger.info("override " + COPROCESSOR_LOCAL_JAR + " to " + path);
System.setProperty(COPROCESSOR_LOCAL_JAR, path);
}
@@ -469,7 +471,6 @@ public class KylinConfig {
return property != null ? property : kylinConfig.getString(prop);
}
-
private String[] getOptionalStringArray(String prop) {
final String property = System.getProperty(prop);
if (!StringUtil.isBlank(property))
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/14141e0d/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 2a09b07..57f1322 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -28,44 +28,77 @@ import org.apache.catalina.core.StandardServer;
import org.apache.catalina.deploy.ErrorPage;
import org.apache.catalina.startup.Tomcat;
import org.apache.hadoop.util.Shell;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.util.ClasspathUtil;
public class DebugTomcat {
+ public static void setupDebugEnv() {
+ try {
+ // test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox
+ ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ overrideDevJobJarLocations();
+
+ System.setProperty("spring.profiles.active", "testing");
+ if (System.getProperty("hdp.version") == null)
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+
+ // workaround for job submission from win to linux -- https://issues.apache.org/jira/browse/MAPREDUCE-4052
+ if (Shell.WINDOWS) {
+ {
+ Field field = Shell.class.getDeclaredField("WINDOWS");
+ field.setAccessible(true);
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(null, false);
+ }
+ {
+ Field field = java.io.File.class.getDeclaredField("pathSeparator");
+ field.setAccessible(true);
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(null, ":");
+ }
+ }
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ private static void overrideDevJobJarLocations() {
+ KylinConfig conf = KylinConfig.getInstanceFromEnv();
+ File devJobJar = findFile("../job/target", "kylin-job-.*-SNAPSHOT-job.jar");
+ if (devJobJar != null) {
+ conf.overrideKylinJobJarPath(devJobJar.getAbsolutePath());
+ }
+ File devCoprocessorJar = findFile("../storage/target", "kylin-storage-.*-SNAPSHOT-coprocessor.jar");
+ if (devCoprocessorJar != null) {
+ conf.overrideCoprocessorLocalJar(devCoprocessorJar.getAbsolutePath());
+ }
+ }
+
+ private static File findFile(String dir, String ptn) {
+ File[] files = new File(dir).listFiles();
+ if (files != null) {
+ for (File f : files) {
+ if (f.getName().matches(ptn))
+ return f;
+ }
+ }
+ return null;
+ }
+
public static void main(String[] args) throws Exception {
+ setupDebugEnv();
+
int port = 7070;
if (args.length >= 1) {
port = Integer.parseInt(args[0]);
}
- // test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox
- ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
-
- // workaround for job submission from win to linux -- https://issues.apache.org/jira/browse/MAPREDUCE-4052
- if (Shell.WINDOWS) {
- {
- Field field = Shell.class.getDeclaredField("WINDOWS");
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
- field.set(null, false);
- }
- {
- Field field = java.io.File.class.getDeclaredField("pathSeparator");
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
- field.set(null, ":");
- }
- }
-
- System.setProperty("spring.profiles.active", "testing");
String webBase = new File("../webapp/app").getAbsolutePath();
if (new File(webBase, "WEB-INF").exists() == false) {
throw new RuntimeException("In order to launch Kylin web app from IDE, please make a symblink from webapp/app/WEB-INF to server/src/main/webapp/WEB-INF");
[04/11] incubator-kylin git commit: added an exists check before
setting column family file permissions
Posted by lu...@apache.org.
added an exists check before setting column family file permissions
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/02562c0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/02562c0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/02562c0d
Branch: refs/heads/0.7
Commit: 02562c0dd7c4010726e3d3ada42c49cb279bd68a
Parents: 7427f74
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:46:58 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:44 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/02562c0d/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
index fa3725c..7a4702e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/BulkLoadJob.java
@@ -72,7 +72,12 @@ public class BulkLoadJob extends AbstractHadoopJob {
FsPermission permission = new FsPermission((short) 0777);
for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
String cfName = cf.getName();
- fs.setPermission(new Path(input + cfName), permission);
+ Path columnFamilyPath = new Path(input + cfName);
+
+ // File may have already been auto-loaded (in the case of MapR DB)
+ if(fs.exists(columnFamilyPath)) {
+ fs.setPermission(columnFamilyPath, permission);
+ }
}
String[] newArgs = new String[2];
[09/11] incubator-kylin git commit: KYLIN-818,
measure returntype support integer ,
and add start/end time to hbase tab in cube, enhance UI in query page
Posted by lu...@apache.org.
KYLIN-818, measure returntype support integer ,and add start/end time to hbase tab in cube,enhance UI in query page
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a7c28c36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a7c28c36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a7c28c36
Branch: refs/heads/0.7
Commit: a7c28c36360ba8c9280b54cc57fca4068cc25355
Parents: 1dc1a37
Author: jiazhong <ji...@ebay.com>
Authored: Mon Jun 8 18:42:02 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Jun 8 18:43:43 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeSchema.js | 2 +-
webapp/app/less/app.less | 2 +-
webapp/app/less/layout.less | 5 +++++
webapp/app/partials/cubes/cube_detail.html | 2 ++
webapp/app/partials/query/query_detail.html | 5 ++---
5 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7c28c36/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index d8e490d..388edaa 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -128,7 +128,7 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
switch ($scope.newMeasure.function.expression) {
case "SUM":
- if (colType === "smallint" || colType === "int" || colType === "bigint") {
+ if (colType === "smallint" || colType === "int" || colType === "bigint"||colType==="integer") {
$scope.newMeasure.function.returntype = 'bigint';
} else {
if (colType.indexOf('decimal') != -1) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7c28c36/webapp/app/less/app.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 5675957..162c929 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -550,7 +550,7 @@ ul.messenger.messenger-fixed.messenger-on-top .messenger-message .messenger-mess
}
/** Enable the text selectable on notification of Messenger **/
-ul.messenger .messenger-message-inner {
+ul.messenger .messenger-message-inner ,.ngCellText{
-moz-user-select: text;
-webkit-user-select: text;
-o-user-select: text;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7c28c36/webapp/app/less/layout.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/layout.less b/webapp/app/less/layout.less
index b3ea6b6..d52d5f5 100644
--- a/webapp/app/less/layout.less
+++ b/webapp/app/less/layout.less
@@ -19,8 +19,13 @@
padding-top: 45px;
}
+body{
+ overflow: auto !important;
+}
+
.main-container {
padding-bottom: 45px;
+ min-width: 960px;
}
.main-content {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7c28c36/webapp/app/partials/cubes/cube_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_detail.html b/webapp/app/partials/cubes/cube_detail.html
index 59f2d69..1fdad18 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -107,6 +107,8 @@
<ul>
<li>Region Count: <span class="red">{{table.regionCount}}</span></li>
<li>Size: <span class="red">{{table.tableSize | bytes:2}}</span></li>
+ <li>Start Time: <span class="red">{{table.dateRangeStart | reverseToGMT0}}</span></li>
+ <li>End Time: <span class="red">{{table.dateRangeEnd | reverseToGMT0}}</span></li>
</ul>
</div>
<div ng-if="cube.hbase">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7c28c36/webapp/app/partials/query/query_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query_detail.html b/webapp/app/partials/query/query_detail.html
index 7157e9c..5ca9010 100644
--- a/webapp/app/partials/query/query_detail.html
+++ b/webapp/app/partials/query/query_detail.html
@@ -128,7 +128,7 @@
</div>
<div ng-if="curQuery.graph.show" class="row">
- <div class="col-xs-2">
+ <div class="col-xs-4 col-md-3">
<div class="graph_content">
<label>Graph Type</label>
@@ -174,8 +174,7 @@
</div>
</div>
</div>
-
- <div class="col-xs-10">
+ <div class="col-xs-8 col-md-9">
<div ng-if="curQuery.graph.data.length > 0">
<div ng-if="curQuery.graph.type.value == 'bar'">
<nvd3-multi-bar-chart
[07/11] incubator-kylin git commit: minor,
KylinConfig read kylin.properties.override, to ease dev env
Posted by lu...@apache.org.
minor, KylinConfig read kylin.properties.override,to ease dev env
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d2ae5035
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d2ae5035
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d2ae5035
Branch: refs/heads/0.7
Commit: d2ae5035f5666e82a7957322d2b4da838168d0ab
Parents: 14141e0
Author: Yang Li <li...@apache.org>
Authored: Sat Jun 6 21:34:43 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jun 6 21:34:43 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfig.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2ae5035/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index b21e63b..08a4d66 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -563,6 +563,23 @@ public class KylinConfig {
logger.error("fail to locate kylin.properties");
throw new RuntimeException("fail to locate kylin.properties");
}
+
+ File overrideFile = new File (propFile.getParentFile(), propFile.getName() + ".override");
+ if (overrideFile.exists()) {
+ try {
+ PropertiesConfiguration conf = new PropertiesConfiguration();
+ conf.load(propFile);
+ PropertiesConfiguration override = new PropertiesConfiguration();
+ override.load(overrideFile);
+ conf.copy(override);
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ conf.save(bout);
+ return new ByteArrayInputStream(bout.toByteArray());
+ } catch (ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
try {
return new FileInputStream(propFile);
} catch (FileNotFoundException e) {
[03/11] incubator-kylin git commit: altered command line pattern
recognition to work with MapRFS
Posted by lu...@apache.org.
altered command line pattern recognition to work with MapRFS
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7427f740
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7427f740
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7427f740
Branch: refs/heads/0.7
Commit: 7427f740e2157e6a6768c6a65f13a0ed3c7d8bc4
Parents: bc8d360
Author: c-horn <ch...@gmail.com>
Authored: Fri May 29 14:35:24 2015 -0400
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 4 17:21:40 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/job/common/ShellExecutable.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7427f740/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index 47a3145..4bc8dac 100644
--- a/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -73,13 +73,13 @@ public class ShellExecutable extends AbstractExecutable {
private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager");
private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)");
private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)");
- private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("HDFS: Number of bytes written=(\\d+)");
+ private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)");
private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)");
- private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("HDFS Read: (\\d+) HDFS Write");
+ private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write");
// hive
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
- private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("HDFS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
@Override
public void log(String message) {