You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/28 06:24:56 UTC
[9/9] kylin git commit: KYLIN-1875 A good snowflake sample data,
and rename 'kylin_metadata' to 'kylin_default_instance'
KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4271f108
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4271f108
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4271f108
Branch: refs/heads/KYLIN-1875
Commit: 4271f108e4134ccf43034fd4a350cf33cd9d2f17
Parents: 3f10b3a
Author: Yang Li <li...@apache.org>
Authored: Sun Nov 27 18:26:33 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Nov 28 14:22:20 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
.../apache/kylin/common/KylinConfigBase.java | 11 +-
.../kylin/metadata/model/DataModelDesc.java | 3 +
.../kylin/metadata/model/MeasureDesc.java | 5 +
.../realization/IRealizationConstants.java | 2 +-
examples/sample_cube/create_sample_tables.sql | 21 +-
.../sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv | 10000 +++++++++
.../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 20000 ++++++++---------
.../template/cube_desc/kylin_sales_cube.json | 157 +-
.../template/model_desc/kylin_sales_model.json | 50 +-
.../template/table/DEFAULT.KYLIN_ACCOUNT.json | 28 +
.../template/table/DEFAULT.KYLIN_SALES.json | 12 +-
.../test_case_data/sandbox/kylin.properties | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 25 +-
.../kylin/rest/security/AclHBaseStorage.java | 1 -
.../rest/security/RealAclHBaseStorage.java | 5 +-
.../apache/kylin/rest/service/QueryService.java | 6 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 2 +-
.../storage/hbase/util/StorageCleanupJob.java | 2 +-
.../apache/kylin/tool/StorageCleanupJob.java | 2 +-
20 files changed, 20246 insertions(+), 10090 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4271f108/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1409394..131a725 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
### METADATA | ENV ###
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+kylin.metadata.url=kylin_default_instance@hbase
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/4271f108/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index aa5e6e1..7dcc771 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -188,13 +188,12 @@ abstract public class KylinConfigBase implements Serializable {
}
public String getMetadataUrlPrefix() {
- String hbaseMetadataUrl = getMetadataUrl();
- String defaultPrefix = "kylin_metadata";
+ String metadataUrl = getMetadataUrl();
+ String defaultPrefix = "kylin_default_instance";
- if (org.apache.commons.lang3.StringUtils.containsIgnoreCase(hbaseMetadataUrl, "@hbase")) {
- int cut = hbaseMetadataUrl.indexOf('@');
- String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut);
- return tmp;
+ if (metadataUrl.endsWith("@hbase")) {
+ int cut = metadataUrl.lastIndexOf('@');
+ return metadataUrl.substring(0, cut);
} else {
return defaultPrefix;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4271f108/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index c2de5d6..ff92def 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -378,6 +378,9 @@ public class DataModelDesc extends RootPersistentEntity {
for (int i = 0; i < pks.length; i++) {
TblColRef col = dimTable.getColumn(pks[i]);
if (col == null) {
+ col = findColumn(pks[i]);
+ }
+ if (col == null || col.getTableRef().equals(dimTable) == false) {
throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity());
}
pkCols[i] = col;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4271f108/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index dcf6350..253b06b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -35,6 +36,8 @@ public class MeasureDesc {
@JsonProperty("function")
private FunctionDesc function;
@JsonProperty("dependent_measure_ref")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @Deprecated
private String dependentMeasureRef;
public String getName() {
@@ -53,10 +56,12 @@ public class MeasureDesc {
this.function = function;
}
+ @Deprecated
public String getDependentMeasureRef() {
return dependentMeasureRef;
}
+ @Deprecated
public void setDependentMeasureRef(String dependentMeasureRef) {
this.dependentMeasureRef = dependentMeasureRef;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4271f108/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 2a3b01e..adc53a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -28,7 +28,7 @@ public class IRealizationConstants {
/**
* For each cube htable, we leverage htable's metadata to keep track of
- * which kylin server(represented by its kylin_metadata prefix) owns this htable
+ * which kylin server(represented by its kylin_default_instance prefix) owns this htable
*/
public final static String HTableTag = "KYLIN_HOST";
http://git-wip-us.apache.org/repos/asf/kylin/blob/4271f108/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index 3189b58..bdfc4d7 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -182,6 +182,19 @@ COUNTRY string
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
+DROP TABLE IF EXISTS DEFAULT.KYLIN_ACCOUNT;
+
+CREATE TABLE DEFAULT.KYLIN_ACCOUNT
+(
+ACCOUNT_ID bigint
+,ACCOUNT_BUYER_LEVEL int COMMENT 'Account Buyer Level'
+,ACCOUNT_SELLER_LEVEL int COMMENT 'Account Seller Level'
+,ACCOUNT_COUNTRY string COMMENT 'Account Country'
+,ACCOUNT_CONTACT string COMMENT 'Account Contact Info'
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE;
+
DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
CREATE TABLE DEFAULT.KYLIN_SALES
@@ -195,16 +208,16 @@ TRANS_ID bigint
,PRICE decimal(19,4) COMMENT 'Order Price'
,ITEM_COUNT bigint COMMENT 'Number of Purchased Goods'
,SELLER_ID bigint COMMENT 'Seller ID'
-,BUYER_COUNTRY string COMMENT 'Buyer Country'
-,SELLER_COUNTRY string COMMENT 'Seller Country'
-,USER_ID string COMMENT 'System User ID'
-,REGION string COMMENT 'System User Region'
+,BUYER_ID bigint COMMENT 'Buyer ID'
+,OPS_USER_ID string COMMENT 'System User ID'
+,OPS_REGION string COMMENT 'System User Region'
)
COMMENT 'Sales order table, fact table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_ACCOUNT;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_COUNTRY;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;