You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/03/11 17:17:18 UTC
[01/14] kylin git commit: KYLIN-2494 ensure model has no dup among
dimensions and measures [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2360 7c16323bd -> 28d66fc65 (forced update)
KYLIN-2494 ensure model has no dup among dimensions and measures
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182ed009
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182ed009
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182ed009
Branch: refs/heads/KYLIN-2360
Commit: 182ed009400629dee1bd7ad0121224cacfd07668
Parents: 2dfffde
Author: Li Yang <li...@apache.org>
Authored: Thu Mar 9 18:03:26 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Mar 9 18:03:26 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ----
.../kylin/metadata/model/DataModelDesc.java | 22 +++++++++-----------
.../kylin/metadata/model/DataModelDescTest.java | 11 ++++++++++
.../test_kylin_inner_join_model_desc.json | 1 +
4 files changed, 22 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 02b2d4b..ad02fe2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -329,10 +329,6 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.cube.allow-appear-in-multiple-projects", "false"));
}
- public boolean allowModelDimensionMetricsOverlap() {
- return Boolean.parseBoolean(getOptional("kylin.model.allow-dimension-metrics-overlap", "false"));
- }
-
// ============================================================================
// JOB
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 1c85f09..0a303ec 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -453,23 +453,21 @@ public class DataModelDesc extends RootPersistentEntity {
}
private boolean validate() {
+
+ // ensure no dup between dimensions/metrics
+ for (ModelDimensionDesc dim : dimensions) {
+ String table = dim.getTable();
+ for (String c : dim.getColumns()) {
+ TblColRef dcol = findColumn(table, c);
+ metrics = ArrayUtils.removeElement(metrics, dcol.getIdentity());
+ }
+ }
+
Set<TblColRef> mcols = new HashSet<>();
for (String m : metrics) {
mcols.add(findColumn(m));
}
- if (!KylinConfig.getInstanceFromEnv().allowModelDimensionMetricsOverlap()) {
- // validate no dup between dimensions/metrics
- for (ModelDimensionDesc dim : dimensions) {
- String table = dim.getTable();
- for (String c : dim.getColumns()) {
- TblColRef dcol = findColumn(table, c);
- if (mcols.contains(dcol))
- throw new IllegalStateException(dcol + " cannot be both dimension and metrics at the same time in " + this);
- }
- }
- }
-
// validate PK/FK are in dimensions
boolean pkfkDimAmended = false;
for (Chain chain : joinsTree.tableChains.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
index 0cfa1f4..b319423 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
@@ -19,7 +19,9 @@
package org.apache.kylin.metadata.model;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.MetadataManager;
@@ -39,6 +41,15 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
public void after() throws Exception {
this.cleanupTestMetadata();
}
+
+ @Test
+ public void testNoDupColInDimAndMeasure() {
+ DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");
+ String[] metrics = model.getMetrics();
+ TblColRef col = model.findColumn("edw.test_cal_dt.cal_dt");
+ assertTrue(metrics.length == 2);
+ assertTrue(ArrayUtils.contains(metrics, col.getIdentity()) == false);
+ }
@Test
public void testGetCopyOf() throws JsonProcessingException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/182ed009/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 49758dd..dbb6df8 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -105,6 +105,7 @@
}
],
"metrics": [
+ "edw.test_cal_dt.cal_dt",
"PRICE",
"ITEM_COUNT"
],
[07/14] kylin git commit: minor, refine kylin config
Posted by bi...@apache.org.
minor, refine kylin config
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0301da01
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0301da01
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0301da01
Branch: refs/heads/KYLIN-2360
Commit: 0301da01b3669d92ad4bd07facfc8f79b7b0e0eb
Parents: 8edf234
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Mar 10 18:42:45 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 17:06:22 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigExt.java | 2 +-
.../security/PasswordPlaceholderConfigurer.java | 19 ++++++++++-
.../apache/kylin/rest/service/AdminService.java | 33 +++++++++++++++++++-
3 files changed, 51 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0301da01/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
index 6062d91..0ad24be 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java
@@ -61,7 +61,7 @@ public class KylinConfigExt extends KylinConfig {
return super.getOptional(prop, dft);
}
- public Properties getAllProperties() {
+ protected Properties getAllProperties() {
Properties result = new Properties();
result.putAll(super.getAllProperties());
result.putAll(overrides);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0301da01/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 092d73a..5bb6d54 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.util.Properties;
@@ -30,6 +31,7 @@ import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigBase;
import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
@@ -51,7 +53,8 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
*/
public PasswordPlaceholderConfigurer() {
Resource[] resources = new Resource[1];
- Properties prop = KylinConfig.getKylinProperties();
+ //Properties prop = KylinConfig.getKylinProperties();
+ Properties prop = getAllKylinProperties();
StringWriter writer = new StringWriter();
prop.list(new PrintWriter(writer));
String propString = writer.getBuffer().toString();
@@ -61,6 +64,20 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
this.setLocations(resources);
}
+ public Properties getAllKylinProperties() {
+ // hack to get all config properties
+ Properties allProps = null;
+ try {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Method getAllMethod = KylinConfigBase.class.getDeclaredMethod("getAllProperties");
+ getAllMethod.setAccessible(true);
+ allProps = (Properties) getAllMethod.invoke(kylinConfig);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return allProps;
+ }
+
public static String encrypt(String strToEncrypt) {
try {
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
http://git-wip-us.apache.org/repos/asf/kylin/blob/0301da01/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index a02804e..4cacb6b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
@@ -27,6 +28,7 @@ import java.util.TreeMap;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.util.OrderedProperties;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -94,7 +96,7 @@ public class AdminService extends BasicService {
logger.debug("Get Kylin Runtime Config");
try {
- return KylinConfig.getInstanceFromEnv().getConfigAsString();
+ return getAllConfigAsString();
} catch (IOException e) {
throw new InternalErrorException("Failed to get Kylin Runtime Config", e);
}
@@ -110,4 +112,33 @@ public class AdminService extends BasicService {
throw new InternalErrorException(e.getMessage(), e);
}
}
+
+ private String getAllConfigAsString() throws IOException {
+ Properties allProps;
+ try {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ Method getAllMethod = KylinConfigBase.class.getDeclaredMethod("getAllProperties");
+ getAllMethod.setAccessible(true);
+ allProps = (Properties) getAllMethod.invoke(kylinConfig);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties();
+
+ final StringBuilder sb = new StringBuilder();
+
+ for (Map.Entry<Object, Object> entry : allProps.entrySet()) {
+ String key = entry.getKey().toString();
+ String value = entry.getValue().toString();
+ if (!orderedProperties.containsProperty(key)) {
+ orderedProperties.setProperty(key, value);
+ } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value))
+ orderedProperties.setProperty(key, value);
+ }
+ for (Map.Entry<String, String> entry : orderedProperties.entrySet()) {
+ sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
+ }
+ return sb.toString();
+ }
}
[09/14] kylin git commit: cube desc json data modification
Posted by bi...@apache.org.
cube desc json data modification
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/791a0100
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/791a0100
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/791a0100
Branch: refs/heads/KYLIN-2360
Commit: 791a010061ad216fd8bb4b5f31f1a1be5a43a141
Parents: 82b2236
Author: luguosheng <55...@qq.com>
Authored: Fri Mar 10 22:44:24 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 17:55:16 2017 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/791a0100/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 7e14d51..cfaadac 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -183,7 +183,7 @@
}
},
"returntype" : "topn(100, 4)",
- "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"}
}
}, {
"name" : "TEST_COUNT_DISTINCT_BITMAP",
http://git-wip-us.apache.org/repos/asf/kylin/blob/791a0100/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index b59d6df..221ffb6 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -183,7 +183,7 @@
}
},
"returntype" : "topn(100, 4)",
- "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
+ "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"}
}
}, {
"name" : "TEST_COUNT_DISTINCT_BITMAP",
[05/14] kylin git commit: KYLIN-2495 query exception when integer
column encoded as date/time encoding
Posted by bi...@apache.org.
KYLIN-2495 query exception when integer column encoded as date/time encoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5f3b228
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5f3b228
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5f3b228
Branch: refs/heads/KYLIN-2360
Commit: c5f3b2286987d12c31310e9c9670d469606e9d4c
Parents: 70dda40
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 10 15:57:10 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 17:17:02 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 16 ++---
.../apache/kylin/cube/model/RowKeyColDesc.java | 7 +--
.../kylin/dimension/AbstractDateDimEnc.java | 25 ++++----
.../org/apache/kylin/dimension/DateDimEnc.java | 64 ++++++++++++++++----
.../org/apache/kylin/dimension/TimeDimEnc.java | 13 ++--
.../kylin/measure/topn/TopNMeasureType.java | 12 +++-
.../apache/kylin/dimension/DateDimEncTest.java | 2 +-
7 files changed, 88 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 2412543..29858f1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -48,6 +48,10 @@ public class DateFormat {
}
return r;
}
+
+ public static String formatToCompactDateStr(long millis) {
+ return formatToDateStr(millis, COMPACT_DATE_PATTERN);
+ }
public static String formatToDateStr(long millis) {
return formatToDateStr(millis, DEFAULT_DATE_PATTERN);
@@ -88,18 +92,6 @@ public class DateFormat {
}
public static long stringToMillis(String str) {
- return stringToMillis(str, null);
- }
-
- public static long stringToMillis(String str, String dateFormat) {
- try {
- if (dateFormat != null) {
- return getDateFormat(dateFormat).parse(str).getTime();
- }
- } catch (ParseException e) {
- // given format does not work, proceed to below
- }
-
// try to be smart and guess the date format
if (isAllDigits(str)) {
if (str.length() == 8)
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 1df73ad..b6f0a27 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -86,10 +86,9 @@ public class RowKeyColDesc implements java.io.Serializable {
encoding = encodingName = TimeDimEnc.ENCODING_NAME;
}
}
- // if (DateDimEnc.ENCODING_NAME.equals(encodingName) && type.isDate() == false)
- // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply date encoding");
- // if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && type.isTimeFamily() == false)
- // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply time encoding");
+
+ encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type);
+
if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily()))
throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
index a54bcda..9c0d0da 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
@@ -18,10 +18,6 @@
package org.apache.kylin.dimension;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -29,20 +25,23 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
public class AbstractDateDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
- interface IMillisCodec extends Serializable {
- long millisToCode(long millis);
+ interface IValueCodec extends Serializable {
+ long valueToCode(String value);
- long codeToMillis(long code);
+ String codeToValue(long code);
}
// ============================================================================
private int fixedLen;
- private IMillisCodec codec;
+ private IValueCodec codec;
- protected AbstractDateDimEnc(int fixedLen, IMillisCodec codec) {
+ protected AbstractDateDimEnc(int fixedLen, IValueCodec codec) {
this.fixedLen = fixedLen;
this.codec = codec;
}
@@ -59,8 +58,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
return;
}
- long millis = DateFormat.stringToMillis(value);
- long code = codec.millisToCode(millis);
+ long code = codec.valueToCode(value);
BytesUtil.writeLong(code, output, outputOffset, fixedLen);
}
@@ -74,8 +72,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
if (code < 0)
throw new IllegalArgumentException();
- long millis = codec.codeToMillis(code);
- return String.valueOf(millis);
+ return codec.codeToValue(code);
}
@Override
@@ -137,7 +134,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
this.fixedLen = in.readInt();
- this.codec = (IMillisCodec) in.readObject();
+ this.codec = (IValueCodec) in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
index 3b390d9..fee6233 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
@@ -20,10 +20,13 @@ package org.apache.kylin.dimension;
import java.io.Serializable;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
+
/**
* This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility.
*/
-public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
+public class DateDimEnc extends AbstractDateDimEnc implements Serializable {
private static final long serialVersionUID = 1L;
public static final int ID_9999_12_31 = 3652426;
@@ -38,24 +41,47 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
@Override
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
- return new DateDimEnc();
+ return new DateDimEnc(args);
}
};
- public DateDimEnc() {
- super(3, new IMillisCodec() {
- private static final long serialVersionUID = 1L;
+ private static class DateDimValueCodec implements IValueCodec {
+
+ private static final long serialVersionUID = 1L;
+ private DataType datatype = null;
- @Override
- public long millisToCode(long millis) {
- return getNumOfDaysSince0000FromMillis(millis);
+ public DateDimValueCodec(String[] args) {
+ if (args != null && args.length == 1) {
+ datatype = DataType.getType(args[0]);
}
+ }
+
+ @Override
+ public long valueToCode(String value) {
+ //if data type is integer, DateFormat.stringToMillis recognizes format like "20001010"
+ long millis = DateFormat.stringToMillis(value);
+
+ return getNumOfDaysSince0000FromMillis(millis);
+ }
- @Override
- public long codeToMillis(long code) {
- return getMillisFromNumOfDaysSince0000(code);
+ @Override
+ public String codeToValue(long code) {
+ long millisFromNumOfDaysSince0000 = getMillisFromNumOfDaysSince0000(code);
+ if (datatype != null && datatype.isIntegerFamily()) {
+ return DateFormat.formatToCompactDateStr(millisFromNumOfDaysSince0000);
+ } else {
+ return String.valueOf(millisFromNumOfDaysSince0000);
}
- });
+ }
+ }
+
+ //keep this for ser/der
+ public DateDimEnc() {
+ super(3, new DateDimValueCodec(null));
+ }
+
+ public DateDimEnc(String[] args) {
+ super(3, new DateDimValueCodec(args));
}
public static long getNumOfDaysSince0000FromMillis(long millis) {
@@ -69,4 +95,18 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
return millis;
}
+ public static String[] replaceEncodingArgs(String encoding, String[] encodingArgs, String encodingName, DataType type) {
+ // https://issues.apache.org/jira/browse/KYLIN-2495
+ if (DateDimEnc.ENCODING_NAME.equals(encodingName)) {
+ if (type.isIntegerFamily()) {
+ if (encodingArgs.length != 0) {
+ throw new IllegalArgumentException("Date encoding should not specify arguments: " + encoding);
+ }
+ return new String[] { type.toString() };
+ }
+ }
+
+ return encodingArgs;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
index 4bef2b8..dcc99dd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
@@ -20,10 +20,12 @@ package org.apache.kylin.dimension;
import java.io.Serializable;
+import org.apache.kylin.common.util.DateFormat;
+
/**
* This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility.
*/
-public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
+public class TimeDimEnc extends AbstractDateDimEnc implements Serializable {
private static final long serialVersionUID = 1L;
public static final String ENCODING_NAME = "time";
@@ -41,17 +43,18 @@ public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
};
public TimeDimEnc() {
- super(4, new IMillisCodec() {
+ super(4, new IValueCodec() {
private static final long serialVersionUID = 1L;
@Override
- public long millisToCode(long millis) {
+ public long valueToCode(String value) {
+ long millis = DateFormat.stringToMillis(value);
return millis / 1000;
}
@Override
- public long codeToMillis(long code) {
- return code * 1000;
+ public String codeToValue(long code) {
+ return String.valueOf(code * 1000);
}
});
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 8c8b5a6..b7252a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -24,10 +24,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DateDimEnc;
import org.apache.kylin.dimension.DictionaryDimEnc;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -47,6 +47,8 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
@@ -155,7 +157,6 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return topNCounter;
}
-
@Override
public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
TopNCounter<ByteArray> topNCounter = value;
@@ -424,7 +425,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
}
Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
- dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion);
+ String encodingName = (String) encodingConf[0];
+ String[] encodingArgs = (String[]) encodingConf[1];
+
+ encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType());
+
+ dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
index 0183b5f..082f3ce 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
@@ -38,7 +38,7 @@ public class DateDimEncTest {
@Before
public void setup() {
- enc = new DateDimEnc();
+ enc = new DateDimEnc(null);
buf = new byte[enc.getLengthOfEncoding()];
}
[10/14] kylin git commit: KYLIN-2499 HybridInstance cache wasn't
cleaned when underlying Cube be updated
Posted by bi...@apache.org.
KYLIN-2499 HybridInstance cache wasn't cleaned when underlying Cube be updated
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d012b9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d012b9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d012b9f
Branch: refs/heads/KYLIN-2360
Commit: 5d012b9fa0ca6b3f98aa80f1ae9418be90d32bca
Parents: 791a010
Author: shaofengshi <sh...@apache.org>
Authored: Sat Mar 11 20:42:15 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 20:42:15 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/hybrid/HybridManager.java | 45 +++++++++++---------
1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d012b9f/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index 2d330c0..cf40416 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -88,14 +89,14 @@ public class HybridManager implements IRealizationProvider {
logger.info("Initializing HybridManager with config " + config);
this.config = config;
this.hybridMap = new CaseInsensitiveStringCache<HybridInstance>(config, "hybrid");
-
+
// touch lower level metadata before registering my listener
reloadAllHybridInstance();
- Broadcaster.getInstance(config).registerListener(new HybridSyncListener(), "hybrid");
+ Broadcaster.getInstance(config).registerListener(new HybridSyncListener(), "hybrid", "cube");
}
private class HybridSyncListener extends Broadcaster.Listener {
-
+
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
clearCache();
@@ -112,15 +113,22 @@ public class HybridManager implements IRealizationProvider {
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException {
- String hybridName = cacheKey;
-
- if (event == Event.DROP)
- hybridMap.removeLocal(hybridName);
- else
- reloadHybridInstance(hybridName);
-
- for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID, hybridName)) {
- broadcaster.notifyProjectSchemaUpdate(prj.getName());
+ if ("hybrid".equals(entity)) {
+ String hybridName = cacheKey;
+
+ if (event == Event.DROP)
+ hybridMap.removeLocal(hybridName);
+ else
+ reloadHybridInstance(hybridName);
+
+ for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID, hybridName)) {
+ broadcaster.notifyProjectSchemaUpdate(prj.getName());
+ }
+ } else if ("cube".equals(entity)) {
+ String cubeName = cacheKey;
+ for (HybridInstance hybrid : getHybridInstancesByChild(RealizationType.CUBE, cubeName)) {
+ reloadHybridInstance(hybrid.getName());
+ }
}
}
}
@@ -139,25 +147,24 @@ public class HybridManager implements IRealizationProvider {
logger.debug("Loaded " + paths.size() + " Hybrid(s)");
}
- public void reloadHybridInstanceByChild(RealizationType type, String realizationName) {
+ public List<HybridInstance> getHybridInstancesByChild(RealizationType type, String realizationName) {
+ List<HybridInstance> result = Lists.newArrayList();
for (HybridInstance hybridInstance : hybridMap.values()) {
- boolean includes = false;
for (RealizationEntry realizationEntry : hybridInstance.getRealizationEntries()) {
if (realizationEntry.getType() == type && realizationEntry.getRealization().equalsIgnoreCase(realizationName)) {
- includes = true;
- break;
+ result.add(hybridInstance);
}
}
- if (includes == true)
- reloadHybridInstance(hybridInstance.getName());
}
+
+ return result;
}
public void reloadHybridInstance(String name) {
reloadHybridInstanceAt(HybridInstance.concatResourcePath(name));
}
-
+
private synchronized HybridInstance reloadHybridInstanceAt(String path) {
ResourceStore store = getStore();
[11/14] kylin git commit: KYLIN-2503 Spark cubing step should show
YARN app link
Posted by bi...@apache.org.
KYLIN-2503 Spark cubing step should show YARN app link
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f38d871a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f38d871a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f38d871a
Branch: refs/heads/KYLIN-2360
Commit: f38d871a72e1e24331e092fd51664fcb8fd47e22
Parents: 5d012b9
Author: shaofengshi <sh...@apache.org>
Authored: Sat Mar 11 22:08:28 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 22:08:28 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/job/common/ShellExecutable.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f38d871a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index a68f242..5e0d0db 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -83,6 +83,10 @@ public class ShellExecutable extends AbstractExecutable {
private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)");
private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
+ // spark
+ private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)");
+ private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)");
+
@Override
public void log(String message) {
Matcher matcher = PATTERN_APP_ID.matcher(message);
@@ -136,6 +140,19 @@ public class ShellExecutable extends AbstractExecutable {
String hdfsWritten = matcher.group(2);
info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
}
+
+ // spark
+ matcher = PATTERN_SPARK_APP_ID.matcher(message);
+ if (matcher.find()) {
+ String app_id = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_ID, app_id);
+ }
+
+ matcher = PATTERN_SPARK_APP_URL.matcher(message);
+ if (matcher.find()) {
+ String trackingUrl = matcher.group(1);
+ info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+ }
}
Map<String, String> getInfo() {
[02/14] kylin git commit: minor changes
Posted by bi...@apache.org.
minor changes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/181890e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/181890e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/181890e5
Branch: refs/heads/KYLIN-2360
Commit: 181890e542783e66078ad707f37dcb273abae596
Parents: 7eb9969
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 9 21:19:26 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 9 21:19:30 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/QueryContext.java | 3 +++
.../org/apache/kylin/rest/service/QueryService.java | 12 ++++--------
2 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/181890e5/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 5457aa5..0b8d519 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -18,6 +18,7 @@
package org.apache.kylin.common;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -38,6 +39,8 @@ public class QueryContext {
private QueryContext() {
// use QueryContext.current() instead
+
+ queryId = UUID.randomUUID().toString();
}
public static QueryContext current() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/181890e5/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 122b823..6a38638 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -41,7 +41,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
@@ -237,7 +236,7 @@ public class QueryService extends BasicService {
public void logQuery(final SQLRequest request, final SQLResponse response) {
final String user = SecurityContextHolder.getContext().getAuthentication().getName();
- final Set<String> realizationNames = new HashSet<String>();
+ final List<String> realizationNames = new LinkedList<>();
final Set<Long> cuboidIds = new HashSet<Long>();
float duration = response.getDuration() / (float) 1000;
boolean storageCacheUsed = response.isStorageCacheUsed();
@@ -251,8 +250,7 @@ public class QueryService extends BasicService {
}
if (ctx.realization != null) {
- String realizationName = ctx.realization.getName();
- realizationNames.add(realizationName);
+ realizationNames.add(ctx.realization.getCanonicalName());
}
}
@@ -332,10 +330,8 @@ public class QueryService extends BasicService {
BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
final QueryContext queryContext = QueryContext.current();
- final String queryId = UUID.randomUUID().toString();
- queryContext.setQueryId(queryId);
- try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
+ try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId())) {
String sql = sqlRequest.getSql();
String project = sqlRequest.getProject();
logger.info("Using project: " + project);
@@ -431,7 +427,7 @@ public class QueryService extends BasicService {
return response;
}
- private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
+ protected void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
checkAuthorization(sqlResponse.getCube());
}
[04/14] kylin git commit: minor,
add missing dependency for tool-assembly module
Posted by bi...@apache.org.
minor, add missing dependency for tool-assembly module
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70dda404
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70dda404
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70dda404
Branch: refs/heads/KYLIN-2360
Commit: 70dda40448bbd5622788dbfea591bf54aa4cdc8a
Parents: 181890e
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 10 14:16:37 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 10 14:16:48 2017 +0800
----------------------------------------------------------------------
build/bin/diag.sh | 2 +-
tool-assembly/pom.xml | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dda404/build/bin/diag.sh
----------------------------------------------------------------------
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
index 5f3cec3..e9354a2 100644
--- a/build/bin/diag.sh
+++ b/build/bin/diag.sh
@@ -47,7 +47,7 @@ then
fi
mkdir -p ${KYLIN_HOME}/ext
- export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+ export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
if [ ${#patient} -eq 36 ]; then
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dda404/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
index 126c0bc..f7faad0 100644
--- a/tool-assembly/pom.xml
+++ b/tool-assembly/pom.xml
@@ -91,6 +91,7 @@
<artifactSet>
<includes>
<!-- shade the httpcore to avoid the lower version conflict with HBase one -->
+ <include>org.apache.httpcomponents:httpclient</include>
<include>org.apache.httpcomponents:httpcore</include>
<include>org.slf4j:slf4j-api</include>
<include>org.slf4j:slf4j-log4j12</include>
[06/14] kylin git commit: KYLIN-2222 web ui uses rest api to decide
which dim encoding is valid for different typed columns
Posted by bi...@apache.org.
KYLIN-2222 web ui uses rest api to decide which dim encoding is valid for different typed columns
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8edf234c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8edf234c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8edf234c
Branch: refs/heads/KYLIN-2360
Commit: 8edf234ce571593145970e4d6cc18934cf761348
Parents: c5f3b22
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 10 15:58:00 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 17:17:11 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/EncodingService.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8edf234c/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
index 7d7d016..60f9974 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.kylin.rest.service;
import java.util.List;
@@ -44,8 +43,8 @@ public class EncodingService extends BasicService {
} else if (dataType.isDateTimeFamily()) {
return Lists.newArrayList(DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME);
} else if (dataType.isStringFamily()) {
- return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, //
- FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME);
+ return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, //
+ FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME);
} else {
throw new IllegalArgumentException("can't provide valid encodings for datatype:" + dataType);
}
[08/14] kylin git commit: minor, refine kylin config
Posted by bi...@apache.org.
minor, refine kylin config
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/82b22364
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/82b22364
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/82b22364
Branch: refs/heads/KYLIN-2360
Commit: 82b223641047a215a0b212f4c3d77e21d1a5203c
Parents: 0301da0
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Mar 11 17:49:23 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 17:49:23 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 20 +++++++++---
.../apache/kylin/rest/service/AdminService.java | 32 +-------------------
2 files changed, 17 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/82b22364/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 4fcc61f..c6b1511 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -228,7 +228,7 @@ public class KylinConfig extends KylinConfigBase {
Properties conf = new Properties();
try {
OrderedProperties orderedProperties = getKylinOrderedProperties();
- for (Map.Entry<String, String> each: orderedProperties.entrySet()) {
+ for (Map.Entry<String, String> each : orderedProperties.entrySet()) {
conf.put(each.getKey(), each.getValue());
}
} catch (FileNotFoundException e) {
@@ -238,7 +238,7 @@ public class KylinConfig extends KylinConfigBase {
return conf;
}
- public static OrderedProperties getKylinOrderedProperties() throws FileNotFoundException {
+ private static OrderedProperties getKylinOrderedProperties() throws FileNotFoundException {
File propFile = getKylinPropertiesFile();
if (propFile == null || !propFile.exists()) {
logger.error("fail to locate " + KYLIN_CONF_PROPERTIES_FILE);
@@ -270,9 +270,21 @@ public class KylinConfig extends KylinConfigBase {
}
}
- public static String getConfigAsString() throws IOException {
- OrderedProperties orderedProperties = getKylinOrderedProperties();
+ public String getConfigAsString() throws IOException {
+ Properties allProps = getAllProperties();
+ OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties();
+
final StringBuilder sb = new StringBuilder();
+
+ for (Map.Entry<Object, Object> entry : allProps.entrySet()) {
+ String key = entry.getKey().toString();
+ String value = entry.getValue().toString();
+ if (!orderedProperties.containsProperty(key)) {
+ orderedProperties.setProperty(key, value);
+ } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value)) {
+ orderedProperties.setProperty(key, value);
+ }
+ }
for (Map.Entry<String, String> entry : orderedProperties.entrySet()) {
sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/82b22364/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index 4cacb6b..66725dc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -20,7 +20,6 @@ package org.apache.kylin.rest.service;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.lang.reflect.Method;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
@@ -28,7 +27,6 @@ import java.util.TreeMap;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.util.OrderedProperties;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -96,7 +94,7 @@ public class AdminService extends BasicService {
logger.debug("Get Kylin Runtime Config");
try {
- return getAllConfigAsString();
+ return KylinConfig.getInstanceFromEnv().getConfigAsString();
} catch (IOException e) {
throw new InternalErrorException("Failed to get Kylin Runtime Config", e);
}
@@ -113,32 +111,4 @@ public class AdminService extends BasicService {
}
}
- private String getAllConfigAsString() throws IOException {
- Properties allProps;
- try {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- Method getAllMethod = KylinConfigBase.class.getDeclaredMethod("getAllProperties");
- getAllMethod.setAccessible(true);
- allProps = (Properties) getAllMethod.invoke(kylinConfig);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties();
-
- final StringBuilder sb = new StringBuilder();
-
- for (Map.Entry<Object, Object> entry : allProps.entrySet()) {
- String key = entry.getKey().toString();
- String value = entry.getValue().toString();
- if (!orderedProperties.containsProperty(key)) {
- orderedProperties.setProperty(key, value);
- } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value))
- orderedProperties.setProperty(key, value);
- }
- for (Map.Entry<String, String> entry : orderedProperties.entrySet()) {
- sb.append(entry.getKey() + "=" + entry.getValue()).append('\n');
- }
- return sb.toString();
- }
}
[13/14] kylin git commit: KYLIN-2305 Unable to use long
searchBase/Pattern for LDAP
Posted by bi...@apache.org.
KYLIN-2305 Unable to use long searchBase/Pattern for LDAP
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/181b6c5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/181b6c5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/181b6c5c
Branch: refs/heads/KYLIN-2360
Commit: 181b6c5c16fc0506a475993f3735a4b7e73a07c1
Parents: 0cb5032
Author: otomarukanta <kk...@yahoo-corp.jp>
Authored: Fri Mar 10 16:34:52 2017 +0900
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Mar 11 22:12:55 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/security/PasswordPlaceholderConfigurer.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/181b6c5c/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 5bb6d54..095b74a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.security;
+import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -51,12 +52,12 @@ public class PasswordPlaceholderConfigurer extends PropertyPlaceholderConfigurer
/**
* The PasswordPlaceholderConfigurer will read Kylin properties as the Spring resource
*/
- public PasswordPlaceholderConfigurer() {
+ public PasswordPlaceholderConfigurer() throws IOException {
Resource[] resources = new Resource[1];
//Properties prop = KylinConfig.getKylinProperties();
Properties prop = getAllKylinProperties();
StringWriter writer = new StringWriter();
- prop.list(new PrintWriter(writer));
+ prop.store(new PrintWriter(writer), "kylin properties");
String propString = writer.getBuffer().toString();
IOUtils.closeQuietly(writer);
InputStream is = IOUtils.toInputStream(propString, Charset.defaultCharset());
[12/14] kylin git commit: KYLIN-2384 backward compatible
Posted by bi...@apache.org.
KYLIN-2384 backward compatible
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0cb5032e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0cb5032e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0cb5032e
Branch: refs/heads/KYLIN-2360
Commit: 0cb5032ee8da30a91f5ef8914138852daead558d
Parents: f38d871
Author: xiefan46 <95...@qq.com>
Authored: Sat Mar 11 16:46:54 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 22:09:55 2017 +0800
----------------------------------------------------------------------
.../kylin/dict/Number2BytesConverter.java | 4 ++
.../org/apache/kylin/dict/NumberDictionary.java | 12 ++++-
.../apache/kylin/dict/NumberDictionary2.java | 7 +++
.../org/apache/kylin/dict/TrieDictionary.java | 48 +++++++++-----------
4 files changed, 44 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
index 4e40527..814c95a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/Number2BytesConverter.java
@@ -53,6 +53,10 @@ public class Number2BytesConverter implements BytesConverter<String>, Serializab
this.maxDigitsBeforeDecimalPoint = maxDigitsBeforeDecimalPoint;
}
+ public void setMaxDigitsBeforeDecimalPoint(int maxDigitsBeforeDecimalPoint) {
+ this.maxDigitsBeforeDecimalPoint = maxDigitsBeforeDecimalPoint;
+ }
+
@Override
public byte[] convertToBytes(String v) {
NumberBytesCodec codec = getCodec(this.maxDigitsBeforeDecimalPoint);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index b987eda..1377e8e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -19,6 +19,8 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.util.ClassUtil;
+
/**
* @author yangli9
*
@@ -32,6 +34,7 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
public NumberDictionary() { // default constructor for Writable interface
super();
+
}
public NumberDictionary(byte[] trieBytes) {
@@ -42,6 +45,13 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
protected boolean isNullObjectForm(T value) {
return value == null || value.equals("");
}
-
+
+ @Override
+ protected void setConverterByName(String converterName) throws Exception {
+ converterName = "org.apache.kylin.dict.Number2BytesConverter";
+ this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ ((Number2BytesConverter)this.bytesConvert).setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY);
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
index 3879d33..0efe936 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
@@ -18,6 +18,8 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.util.ClassUtil;
+
/**
* This class uses MAX_DIGITS_BEFORE_DECIMAL_POINT (=19) instead of legacy (=16).
*/
@@ -35,5 +37,10 @@ public class NumberDictionary2<T> extends NumberDictionary<T> {
super(trieBytes);
}
+ @Override
+ protected void setConverterByName(String converterName) throws Exception {
+ converterName = "org.apache.kylin.dict.Number2BytesConverter";
+ this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0cb5032e/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 9b84734..8849015 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -42,21 +42,21 @@ import com.google.common.base.Preconditions;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[] to
* int IDs.
- *
+ * <p>
* With Trie the memory footprint of the mapping is kinda minimized at the cost
* CPU, if compared to HashMap of ID Arrays. Performance test shows Trie is
* roughly 10 times slower, so there's a cache layer overlays on top of Trie and
* gracefully fall back to Trie using a weak reference.
- *
+ * <p>
* The implementation is thread-safe.
- *
+ *
* @author yangli9
*/
-@SuppressWarnings({ "rawtypes", "unchecked" })
+@SuppressWarnings({"rawtypes", "unchecked"})
public class TrieDictionary<T> extends CacheDictionary<T> {
private static final long serialVersionUID = 1L;
- public static final byte[] MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
+ public static final byte[] MAGIC = new byte[]{0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74}; // "TrieDict"
public static final int MAGIC_SIZE_I = MAGIC.length;
public static final int BIT_IS_LAST_CHILD = 0x80;
@@ -104,7 +104,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
String converterName = headIn.readUTF();
if (converterName.isEmpty() == false)
- this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ setConverterByName(converterName);
this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
@@ -119,6 +119,10 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
}
}
+ protected void setConverterByName(String converterName) throws Exception {
+ this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
+ }
+
@Override
public int getMinId() {
return baseId;
@@ -151,19 +155,14 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
/**
* returns a code point from [0, nValues), preserving order of value
- *
- * @param n
- * -- the offset of current node
- * @param inp
- * -- input value bytes to lookup
- * @param o
- * -- offset in the input value bytes matched so far
- * @param inpEnd
- * -- end of input
- * @param roundingFlag
- * -- =0: return -1 if not found
- * -- <0: return closest smaller if not found, return -1
- * -- >0: return closest bigger if not found, return nValues
+ *
+ * @param n -- the offset of current node
+ * @param inp -- input value bytes to lookup
+ * @param o -- offset in the input value bytes matched so far
+ * @param inpEnd -- end of input
+ * @param roundingFlag -- =0: return -1 if not found
+ * -- <0: return closest smaller if not found, return -1
+ * -- >0: return closest bigger if not found, return nValues
*/
private int lookupSeqNoFromValue(int n, byte[] inp, int o, int inpEnd, int roundingFlag) {
if (o == inpEnd) // special 'empty' value
@@ -257,13 +256,10 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
/**
* returns a code point from [0, nValues), preserving order of value, or -1
* if not found
- *
- * @param n
- * -- the offset of current node
- * @param seq
- * -- the code point under track
- * @param returnValue
- * -- where return value is written to
+ *
+ * @param n -- the offset of current node
+ * @param seq -- the code point under track
+ * @param returnValue -- where return value is written to
*/
private int lookupValueFromSeqNo(int n, int seq, byte[] returnValue, int offset) {
int o = offset;
[03/14] kylin git commit: KYLIN-2493 Auto enlarge buffer in
FactDistinctColumnsMapper
Posted by bi...@apache.org.
KYLIN-2493 Auto enlarge buffer in FactDistinctColumnsMapper
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7eb9969e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7eb9969e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7eb9969e
Branch: refs/heads/KYLIN-2360
Commit: 7eb9969e1ce54cc919ad47f2589d85da44090d18
Parents: 182ed00
Author: xiefan46 <95...@qq.com>
Authored: Thu Mar 9 18:25:11 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 9 21:19:30 2017 +0800
----------------------------------------------------------------------
.../mr/steps/FactDistinctColumnsMapper.java | 24 +++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7eb9969e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index d9c1309..07e788c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -48,7 +48,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
public static enum RawDataCounter {
BYTES
- };
+ }
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
@@ -153,8 +153,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
tmpbuf.clear();
+ byte[] valueBytes = Bytes.toBytes(fieldValue);
+ int size = valueBytes.length + 1;
+ if (size >= tmpbuf.capacity()) {
+ tmpbuf = ByteBuffer.allocate(countNewSize(tmpbuf.capacity(), size));
+ }
tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
- tmpbuf.put(Bytes.toBytes(fieldValue));
+ tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
sortableKey.setText(outputKey);
//judge type
@@ -176,8 +181,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
String fieldValue = row[partitionColumnIndex];
if (fieldValue != null) {
tmpbuf.clear();
+ byte[] valueBytes = Bytes.toBytes(fieldValue);
+ int size = valueBytes.length + 1;
+ if (size >= tmpbuf.capacity()) {
+ tmpbuf = ByteBuffer.allocate(countNewSize(tmpbuf.capacity(), size));
+ }
tmpbuf.put(MARK_FOR_PARTITION_COL);
- tmpbuf.put(Bytes.toBytes(fieldValue));
+ tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
sortableKey.setText(outputKey);
sortableKey.setTypeId((byte) 0);
@@ -243,4 +253,12 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
}
}
+
+ private int countNewSize(int oldSize, int dataSize) {
+ int newSize = oldSize * 2;
+ while (newSize < dataSize) {
+ newSize = newSize * 2;
+ }
+ return newSize;
+ }
}
[14/14] kylin git commit: KYLIN-2360 fix bugs reported by Sonar
Posted by bi...@apache.org.
KYLIN-2360 fix bugs reported by Sonar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28d66fc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28d66fc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28d66fc6
Branch: refs/heads/KYLIN-2360
Commit: 28d66fc6565e48c95dbf15b3ddc2602c4872a89b
Parents: 181b6c5
Author: Billy Liu <bi...@apache.org>
Authored: Wed Mar 8 22:10:46 2017 -0800
Committer: Billy Liu <bi...@apache.org>
Committed: Sat Mar 11 09:17:02 2017 -0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 35 --------------------
.../kylin/common/util/HiveCmdBuilder.java | 5 ++-
.../apache/kylin/common/util/SparkEntry.java | 2 +-
.../apache/kylin/common/util/ZipFileUtils.java | 10 ++++--
.../java/org/apache/kylin/cube/CubeSegment.java | 3 +-
.../org/apache/kylin/cube/CubeValidator.java | 4 +++
.../org/apache/kylin/cube/kv/CubeDimEncMap.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../validation/rule/AggregationGroupRule.java | 2 +-
.../kylin/dimension/IDimensionEncodingMap.java | 2 +-
.../apache/kylin/metadata/MetadataManager.java | 5 ++-
.../metadata/model/IJoinedFlatTableDesc.java | 2 +-
.../apache/kylin/metadata/model/JoinsTree.java | 2 +-
.../kylin/metadata/model/PartitionDesc.java | 2 +-
.../spark/SparkBatchCubingJobBuilder2.java | 6 ++--
.../kylin/engine/spark/SparkExecutable.java | 3 +-
.../kylin/query/schema/OLAPSchemaFactory.java | 6 +++-
.../rest/controller/StreamingController.java | 3 ++
.../kylin/storage/hbase/util/RowCounterCLI.java | 30 ++++++++++-------
.../kylin/storage/hdfs/HDFSResourceStore.java | 4 +--
.../apache/kylin/tool/CubeMetaExtractor.java | 2 +-
21 files changed, 61 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index c6b1511..9c7e12e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -24,10 +24,8 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.PrintWriter;
import java.io.StringReader;
import java.nio.charset.Charset;
-import java.util.Enumeration;
import java.util.Map;
import java.util.Properties;
@@ -361,39 +359,6 @@ public class KylinConfig extends KylinConfigBase {
return this.base() == ((KylinConfig) another).base();
}
- public static void writeOverrideProperties(Properties properties) throws IOException {
- File propFile = getKylinPropertiesFile();
- File overrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override");
- overrideFile.createNewFile();
- FileInputStream fis2 = null;
- Properties override = new Properties();
- try {
- fis2 = new FileInputStream(overrideFile);
- override.load(fis2);
- for (Map.Entry<Object, Object> entries : properties.entrySet()) {
- override.setProperty(entries.getKey().toString(), entries.getValue().toString());
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- IOUtils.closeQuietly(fis2);
- }
-
- PrintWriter pw = null;
- try {
- pw = new PrintWriter(overrideFile);
- Enumeration<?> e = override.propertyNames();
- while (e.hasMoreElements()) {
- String key = (String) e.nextElement();
- pw.println(key + "=" + override.getProperty(key));
- }
- pw.close();
- } finally {
- IOUtils.closeQuietly(pw);
- }
-
- }
-
private static void dumpStackTrace() {
//uncomment below to start debugging
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 5942ba9..0ec2d36 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -62,9 +62,11 @@ public class HiveCmdBuilder {
break;
case BEELINE:
BufferedWriter bw = null;
+ FileWriter fw = null;
try {
File tmpHql = File.createTempFile("beeline_", ".hql");
- bw = new BufferedWriter(new FileWriter(tmpHql));
+ fw = new FileWriter(tmpHql);
+ bw = new BufferedWriter(fw);
for (String statement : statements) {
bw.write(statement);
bw.newLine();
@@ -84,6 +86,7 @@ public class HiveCmdBuilder {
throw new RuntimeException(e);
} finally {
IOUtils.closeQuietly(bw);
+ IOUtils.closeQuietly(fw);
}
break;
default:
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
index ce11b91..b26959f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
@@ -28,7 +28,7 @@ public final class SparkEntry {
if (!(args.length >= 2)) {
throw new IllegalArgumentException(String.valueOf("-className is required"));
}
- if (!(args[0].equals("-className"))) {
+ if (!("-className".equals(args[0]))) {
throw new IllegalArgumentException(String.valueOf("-className is required"));
}
final String className = args[1];
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
index a2326ee..6f6d13d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
@@ -40,18 +40,23 @@ public class ZipFileUtils {
throw new RuntimeException("Zipfile must end with .zip");
}
ZipOutputStream zipFile = null;
+ FileOutputStream fos = null;
try {
- zipFile = new ZipOutputStream(new FileOutputStream(zipFilename));
+ fos = new FileOutputStream(zipFilename);
+ zipFile = new ZipOutputStream(fos);
compressDirectoryToZipfile(normDir(new File(sourceDir).getParent()), normDir(sourceDir), zipFile);
} finally {
IOUtils.closeQuietly(zipFile);
+ IOUtils.closeQuietly(fos);
}
}
public static void decompressZipfileToDirectory(String zipFileName, File outputFolder) throws IOException {
ZipInputStream zipInputStream = null;
+ FileInputStream fis = null;
try {
- zipInputStream = new ZipInputStream(new FileInputStream(zipFileName));
+ fis = new FileInputStream(zipFileName);
+ zipInputStream = new ZipInputStream(fis);
ZipEntry zipEntry = null;
while ((zipEntry = zipInputStream.getNextEntry()) != null) {
logger.info("decompressing " + zipEntry.getName() + " is directory:" + zipEntry.isDirectory() + " available: " + zipInputStream.available());
@@ -73,6 +78,7 @@ public class ZipFileUtils {
}
} finally {
IOUtils.closeQuietly(zipInputStream);
+ IOUtils.closeQuietly(fis);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 45310f0..835accc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -19,6 +19,7 @@
package org.apache.kylin.cube;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
@@ -99,7 +100,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
@JsonProperty("rowkey_stats")
- private List<Object[]> rowkeyStats = Lists.newArrayList();
+ private ArrayList<Object[]> rowkeyStats = Lists.newArrayList();
@JsonProperty("source_partition_offset_start")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-cube/src/main/java/org/apache/kylin/cube/CubeValidator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeValidator.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeValidator.java
index f94752f..7fbfca0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeValidator.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeValidator.java
@@ -95,6 +95,10 @@ public class CubeValidator {
// for all other segments, sourceOffset SHOULD fit/connect other segments
for (CubeSegment seg : news) {
Pair<Boolean, Boolean> pair = fitInSegments(all, seg);
+ if(pair == null) {
+ logger.warn("Empty segment found, ignore");
+ continue;
+ }
boolean startFit = pair.getFirst();
boolean endFit = pair.getSecond();
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
index bd9554a..84048ecb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
import java.util.Map;
-public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializable {
+public class CubeDimEncMap implements IDimensionEncodingMap {
private static final Logger logger = LoggerFactory.getLogger(CubeDimEncMap.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index c1469fe..f0955b5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -440,7 +440,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
public int getBuildLevel() {
- if (aggregationGroups == null || aggregationGroups.size() == 0)
+ if (aggregationGroups == null || aggregationGroups.isEmpty())
throw new IllegalStateException("Cube has no aggregation group.");
return Collections.max(Collections2.transform(aggregationGroups, new Function<AggregationGroup, Integer>() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index c9e2d28..5d51bc7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -50,7 +50,7 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
private void inner(CubeDesc cube, ValidateContext context) {
- if (cube.getAggregationGroups() == null || cube.getAggregationGroups().size() == 0) {
+ if (cube.getAggregationGroups() == null || cube.getAggregationGroups().isEmpty()) {
context.addResult(ResultLevel.ERROR, "Cube should have at least one Aggregation group.");
return;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-metadata/src/main/java/org/apache/kylin/dimension/IDimensionEncodingMap.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IDimensionEncodingMap.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IDimensionEncodingMap.java
index 4df4701..fb25c1e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IDimensionEncodingMap.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IDimensionEncodingMap.java
@@ -30,7 +30,7 @@ import org.apache.kylin.metadata.model.TblColRef;
* hard to enumerate all values of a encoding, thus TupleFilterDictionaryTranslater
* cannot work on DimensionEncoding.
*/
-public interface IDimensionEncodingMap {
+public interface IDimensionEncodingMap extends java.io.Serializable {
/** Get dimension encoding of a column */
DimensionEncoding get(TblColRef col);
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9427ace..8a02c59 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -261,15 +261,14 @@ public class MetadataManager {
String path = desc.getResourcePath();
getStore().putResource(path, desc, EXTERNAL_FILTER_DESC_SERIALIZER);
desc = reloadExternalFilterAt(path);
- extFilterMap.put(desc.getName(), desc);
-
+ if(desc != null)
+ extFilterMap.put(desc.getName(), desc);
}
public void removeExternalFilter(String name) throws IOException {
String path = ExternalFilterDesc.concatResourcePath(name);
getStore().deleteResource(path);
extFilterMap.remove(name);
-
}
private void init(KylinConfig config) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
index b545e50..509cd46 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
*/
-public interface IJoinedFlatTableDesc {
+public interface IJoinedFlatTableDesc extends java.io.Serializable{
String getTableName();
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index c7666cb..3efca2a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -30,7 +30,7 @@ import com.google.common.base.Preconditions;
public class JoinsTree implements Serializable {
private static final long serialVersionUID = 1L;
- final Map<String, Chain> tableChains = new LinkedHashMap<>();
+ final HashMap<String, Chain> tableChains = new LinkedHashMap<>();
public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
for (JoinDesc join : joins) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 38c2de5..19c58fd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -173,7 +173,7 @@ public class PartitionDesc implements Serializable {
// ============================================================================
- public static interface IPartitionConditionBuilder {
+ public static interface IPartitionConditionBuilder extends java.io.Serializable{
String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index e0b3e6c..a72999e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -74,11 +74,11 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
}
- private String findJar(String className, String perferLibraryName) {
+ private String findJar(String className, String preferLibraryName) {
try {
- return ClassUtil.findContainingJar(Class.forName(className), perferLibraryName);
+ return ClassUtil.findContainingJar(Class.forName(className), preferLibraryName);
} catch (ClassNotFoundException e) {
- logger.warn("failed to locate jar for class " + className + ", ignore it");
+ logger.warn("failed to locate jar for class " + className + ", ignore it", e);
}
return "";
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 5ad21cf..145e3ff 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -80,8 +80,7 @@ public class SparkExecutable extends AbstractExecutable {
String jars = this.getParam(JARS);
//hadoop conf dir
- String hadoopConf = null;
- hadoopConf = System.getProperty("kylin.hadoop.conf.dir");
+ String hadoopConf = System.getProperty("kylin.hadoop.conf.dir");
if (StringUtils.isEmpty(hadoopConf)) {
throw new RuntimeException("kylin_hadoop_conf_dir is empty, check if there's error in the output of 'kylin.sh start'");
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index 132e65a..ffa2ee7 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -33,6 +33,7 @@ import org.apache.calcite.schema.SchemaFactory;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.util.ConversionUtil;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.measure.MeasureTypeFactory;
@@ -101,10 +102,11 @@ public class OLAPSchemaFactory implements SchemaFactory {
}
}
+ FileWriter out = null;
try {
File tmp = File.createTempFile("olap_model_", ".json");
- FileWriter out = new FileWriter(tmp);
+ out = new FileWriter(tmp);
out.write("{\n");
out.write(" \"version\": \"1.0\",\n");
out.write(" \"defaultSchema\": \"" + majoritySchemaName + "\",\n");
@@ -138,6 +140,8 @@ public class OLAPSchemaFactory implements SchemaFactory {
} catch (IOException e) {
throw new RuntimeException(e);
+ } finally {
+ IOUtils.closeQuietly(out);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index 407ee2e..6e096b0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -108,6 +108,9 @@ public class StreamingController extends BasicController {
StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
+ if(streamingConfig == null || kafkaConfig == null){
+ return streamingRequest;
+ }
boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index db516bb..ed6da72 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -21,6 +21,7 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import java.util.Iterator;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
@@ -43,6 +44,7 @@ public class RowCounterCLI {
if (args == null || args.length != 3) {
System.out.println("Usage: hbase org.apache.hadoop.util.RunJar kylin-job-latest.jar org.apache.kylin.job.tools.RowCounterCLI [HTABLE_NAME] [STARTKEY] [ENDKEY]");
+ return;
}
System.out.println(args[0]);
@@ -71,20 +73,24 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- Connection conn = ConnectionFactory.createConnection(conf);
- Table tableInterface = conn.getTable(TableName.valueOf(htableName));
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
- Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
- int counter = 0;
- while (iterator.hasNext()) {
- iterator.next();
- counter++;
- if (counter % 1000 == 1) {
- System.out.println("number of rows: " + counter);
+ Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
+ int counter = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ counter++;
+ if (counter % 1000 == 1) {
+ System.out.println("number of rows: " + counter);
+ }
}
+ System.out.println("number of rows: " + counter);
+ tableInterface.close();
+ }finally {
+ IOUtils.closeQuietly(conn);
}
- System.out.println("number of rows: " + counter);
- tableInterface.close();
- conn.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index 6744805..31f68e8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -49,7 +49,7 @@ public class HDFSResourceStore extends ResourceStore {
private FileSystem fs;
- public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
+ public HDFSResourceStore(KylinConfig kylinConfig) throws IOException {
super(kylinConfig);
String metadataUrl = kylinConfig.getMetadataUrl();
int cut = metadataUrl.indexOf('@');
@@ -73,7 +73,7 @@ public class HDFSResourceStore extends ResourceStore {
}
- private void createMetaFolder(Path metaDirName, KylinConfig kylinConfig) throws Exception {
+ private void createMetaFolder(Path metaDirName, KylinConfig kylinConfig) throws IOException {
//create hdfs meta path
if (!fs.exists(metaDirName)) {
fs.mkdirs(metaDirName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/28d66fc6/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 35831f7..4e80708 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -274,7 +274,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
objectMapper.writeValue(f, rootNode);
}
} catch (JsonProcessingException ex) {
- logger.info("cannot parse file {}", f);
+ logger.info("cannot parse file {}", f, ex);
}
}