You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/16 09:36:56 UTC
kylin git commit: KYLIN-2450 sample.sh generates sample data to
kylin.source.hive.database-for-flat-table
Repository: kylin
Updated Branches:
refs/heads/master e95c07156 -> a60f893ff
KYLIN-2450 sample.sh generates sample data to kylin.source.hive.database-for-flat-table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a60f893f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a60f893f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a60f893f
Branch: refs/heads/master
Commit: a60f893ff0c6d8f1e34ef9bbdd3cab7cc875aa99
Parents: e95c071
Author: Billy Liu <bi...@apache.org>
Authored: Thu Feb 16 17:36:49 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Feb 16 17:36:49 2017 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 16 +++++++++--
build/conf/kylin.properties | 2 +-
.../org/apache/kylin/common/util/Bytes.java | 10 +++----
.../kylin/common/util/CliCommandExecutor.java | 1 +
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 2 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../kylin/engine/mr/common/NDCuboidBuilder.java | 2 +-
.../steps/RowKeyDistributionCheckerMapper.java | 2 +-
examples/sample_cube/create_sample_tables.sql | 30 ++++++++++----------
.../apache/kylin/jdbc/json/SQLResponseStub.java | 2 +-
.../apache/kylin/source/hive/SchemaChecker.java | 2 +-
11 files changed, 41 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 1363a5d..74991c0 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -22,7 +22,6 @@ source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
source ${dir}/check-env.sh "if-not-yet"
job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
-echo "Going to create sample tables in hive..."
cd ${KYLIN_HOME}/sample_cube/data
echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
@@ -30,12 +29,17 @@ hadoop fs -mkdir -p /tmp/kylin/sample_cube/data
hadoop fs -put * /tmp/kylin/sample_cube/data/
hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
+sample_database=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.database-for-flat-table`
+echo "Going to create sample tables in hive to database "$sample_database" by "$hive_client_mode
if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
- beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+ beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$sample_database
+#### Caution: hard code the beeline parameter by appending sample database.
+ beeline ${beeline_params}"/"$sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
- hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+ hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
+ hive --database ${sample_database} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
fi
echo "Sample hive tables are created successfully; Going to create sample cube..."
@@ -54,6 +58,12 @@ cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
sed -i "s/%default_storage_type%/${default_storage_type}/g" ${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
sed -i "s/%default_engine_type%/${default_engine_type}/g" ${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/DEFAULT./${sample_database}./g" ${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube.json
+sed -i "s/DEFAULT./${sample_database}./g" ${KYLIN_HOME}/sample_cube/metadata/model_desc/kylin_sales_model.json
+sed -i "s/DEFAULT./${sample_database}./g" ${KYLIN_HOME}/sample_cube/metadata/project/learn_kylin.json
+
+cd ${KYLIN_HOME}/sample_cube/metadata/table
+ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & ${sample_database}\2.json/"|sh -v
cd ${KYLIN_HOME}
hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index f2ebb80..12c4c9c 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -47,7 +47,7 @@ kylin.web.cross-domain-enabled=true
kylin.source.hive.client=cli
# Parameters for beeline client, only necessary if hive client is beeline
-#kylin.source.hive.beeline-params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000'
+#kylin.source.hive.beeline-params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u jdbc:hive2://localhost:10000
kylin.source.hive.keep-flat-table=false
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java b/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
index f64f4b2..fc501ea 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Bytes.java
@@ -328,7 +328,7 @@ public class Bytes {
continue;
}
// turn hex ASCII digit -> number
- byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2));
+ byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + (toBinaryFromHex((byte) hd2)) & 0xff);
b[size++] = d;
i += 3; // skip 3
@@ -1118,8 +1118,8 @@ public class Bytes {
}
final int minLength = Math.min(length1, length2);
final int minWords = minLength / SIZEOF_LONG;
- final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
- final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
+ final long offset1Adj = offset1 + (long)BYTE_ARRAY_BASE_OFFSET;
+ final long offset2Adj = offset2 + (long)BYTE_ARRAY_BASE_OFFSET;
/*
* Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
@@ -1455,7 +1455,7 @@ public class Bytes {
if (inclusive) {
diffBI = diffBI.add(BigInteger.ONE);
}
- final BigInteger splitsBI = BigInteger.valueOf(num + 1);
+ final BigInteger splitsBI = BigInteger.valueOf(num + 1L);
if (diffBI.compareTo(splitsBI) < 0) {
return null;
}
@@ -1942,7 +1942,7 @@ public class Bytes {
hex = hex.toUpperCase();
byte[] b = new byte[hex.length() / 2];
for (int i = 0; i < b.length; i++) {
- b[i] = (byte) ((toBinaryFromHex((byte) hex.charAt(2 * i)) << 4) + toBinaryFromHex((byte) hex.charAt((2 * i + 1))));
+ b[i] = (byte) ((toBinaryFromHex((byte) hex.charAt(2 * i)) << 4) + (toBinaryFromHex((byte) hex.charAt((2 * i + 1))) & 0xff));
}
return b;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index f97b609..9d41c86 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -142,6 +142,7 @@ public class CliCommandExecutor {
int exitCode = proc.waitFor();
return Pair.newPair(exitCode, result.toString());
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index a0b4d46..8557acf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -41,7 +41,7 @@ public final class InMemCubeBuilderUtils {
ImmutableBitSet measureColumns = parentDimensionAndMetricColumnBitSet.getSecond();
ImmutableBitSet childDimensions = parentDimensions;
long mask = Long.highestOneBit(baseCuboidId);
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
+ long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
int index = 0;
for (int i = 0; i < parentCuboidIdActualLength; i++) {
if ((mask & baseCuboidId) > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index dbe4554..9cb60f8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -201,7 +201,7 @@ public class CubeStatsReader {
KylinConfig kylinConf = cubeSegment.getConfig();
long mask = Long.highestOneBit(baseCuboidId);
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
+ long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
for (int i = 0; i < parentCuboidIdActualLength; i++) {
if ((mask & cuboidId) > 0) {
rowkeyLength += rowKeyColumnLength.get(i); //colIO.getColumnLength(columnList.get(i));
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
index 4e98618..9ab42ea 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
@@ -67,7 +67,7 @@ public class NDCuboidBuilder implements Serializable {
long mask = Long.highestOneBit(parentCuboid.getId());
long parentCuboidId = parentCuboid.getId();
long childCuboidId = childCuboid.getId();
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
+ long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
int index = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId
for (int i = 0; i < parentCuboidIdActualLength; i++) {
if ((mask & parentCuboidId) > 0) {// if the this bit position equals
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index a2efe04..eab57d1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -68,7 +68,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Tex
for (Text t : keyList) {
if (key.compareTo(t) < 0) {
Long v = resultMap.get(t);
- long length = key.getLength() + value.getLength();
+ long length = (long)key.getLength() + value.getLength();
v += length;
resultMap.put(t, v);
break;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index bdfc4d7..24f26f8 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -16,9 +16,9 @@
-- limitations under the License.
--
-DROP TABLE IF EXISTS DEFAULT.KYLIN_CAL_DT;
+DROP TABLE IF EXISTS KYLIN_CAL_DT;
-CREATE TABLE DEFAULT.KYLIN_CAL_DT
+CREATE TABLE KYLIN_CAL_DT
(
CAL_DT date COMMENT 'Date, PK'
,YEAR_BEG_DT date COMMENT 'YEAR Begin Date'
@@ -125,9 +125,9 @@ COMMENT 'Date Dimension Table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-DROP TABLE IF EXISTS DEFAULT.KYLIN_CATEGORY_GROUPINGS;
+DROP TABLE IF EXISTS KYLIN_CATEGORY_GROUPINGS;
-CREATE TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS
+CREATE TABLE KYLIN_CATEGORY_GROUPINGS
(
LEAF_CATEG_ID bigint COMMENT 'Category ID, PK'
,LEAF_CATEG_NAME string
@@ -170,9 +170,9 @@ COMMENT 'Detail category inforamtion, Dimension Table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-DROP TABLE IF EXISTS DEFAULT.KYLIN_COUNTRY;
+DROP TABLE IF EXISTS KYLIN_COUNTRY;
-CREATE TABLE DEFAULT.KYLIN_COUNTRY
+CREATE TABLE KYLIN_COUNTRY
(
COUNTRY string
,LATITUDE double
@@ -182,9 +182,9 @@ COUNTRY string
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-DROP TABLE IF EXISTS DEFAULT.KYLIN_ACCOUNT;
+DROP TABLE IF EXISTS KYLIN_ACCOUNT;
-CREATE TABLE DEFAULT.KYLIN_ACCOUNT
+CREATE TABLE KYLIN_ACCOUNT
(
ACCOUNT_ID bigint
,ACCOUNT_BUYER_LEVEL int COMMENT 'Account Buyer Level'
@@ -195,9 +195,9 @@ ACCOUNT_ID bigint
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
+DROP TABLE IF EXISTS KYLIN_SALES;
-CREATE TABLE DEFAULT.KYLIN_SALES
+CREATE TABLE KYLIN_SALES
(
TRANS_ID bigint
,PART_DT date COMMENT 'Order Date'
@@ -216,8 +216,8 @@ COMMENT 'Sales order table, fact table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_ACCOUNT;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_COUNTRY;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
-LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE KYLIN_SALES;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE KYLIN_ACCOUNT;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE KYLIN_COUNTRY;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE KYLIN_CAL_DT;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE KYLIN_CATEGORY_GROUPINGS;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
index a57177c..a05b6d6 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
@@ -154,7 +154,7 @@ public class SQLResponseStub implements Serializable {
}
@JsonIgnoreProperties(ignoreUnknown = true)
- public static class ColumnMetaStub {
+ public static class ColumnMetaStub implements Serializable{
private boolean isAutoIncrement;
private boolean isCaseSensitive;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a60f893f/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index e2a94fb..3488fbd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -81,7 +81,7 @@ public class SchemaChecker {
buf.append("- ").append(reason).append("\n");
}
- return new CheckResult(false, format("Found %d issue(s) with '%s':\n%s Please disable and purge related cube(s) first", reasons.size(), tableName, buf.toString()));
+ return new CheckResult(false, format("Found %d issue(s) with '%s':%n%s Please disable and purge related cube(s) first", reasons.size(), tableName, buf.toString()));
}
}