You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/04/24 03:49:22 UTC
[kylin] branch master updated (5ab78cb -> b1c8944)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 5ab78cb KYLIN-3343 support RDBMS on GUI (#131)
new 6fb75b6 KYLIN-3331 Add script check-hive-usability.sh to avoid script hangs
new 9509989 KYLIN-3328 Upgrade the metadata of sample cube to latest
new 9d88e47 minor, fix exception swallow in CubeDesc.init()
new 71397fd KYLIN-3289 Refactor the storage garbage clean up code
new a9dcad1 KYLIN-3337 Use an auto-closeable style API to replace KylinConfig.setKylinConfigThreadLocal()
new 143cfdc Minor, add config for enable and disable auto merge
new b1c8944 KYLIN-3340, enhance hql check
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../{get-properties.sh => check-hive-usability.sh} | 29 +-
build/bin/find-hive-dependency.sh | 1 +
.../java/org/apache/kylin/common/KylinConfig.java | 31 +-
.../org/apache/kylin/common/KylinConfigBase.java | 6 +-
.../kylin/common/util/AbstractApplication.java | 2 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 4 +
.../java/org/apache/kylin/common/util/Pair.java | 8 +
.../org/apache/kylin/common/KylinConfigTest.java | 32 +-
.../org/apache/kylin/cube/CubeDescManager.java | 6 +-
.../java/org/apache/kylin/cube/CubeInstance.java | 2 +-
.../java/org/apache/kylin/cube/model/CubeDesc.java | 3 +-
.../kylin/engine/mr/common/AbstractHadoopJob.java | 8 +-
.../template/cube/kylin_sales_cube.json | 4 +-
.../template/cube/kylin_streaming_cube.json | 1 +
.../template/cube_desc/kylin_sales_cube.json | 14 +-
.../template/cube_desc/kylin_streaming_cube.json | 2 +-
.../template/model_desc/kylin_sales_model.json | 41 ++-
.../sample_cube/template/project/learn_kylin.json | 4 +-
.../table/DEFAULT.KYLIN_ACCOUNT--learn_kylin.json | 4 +-
.../table/DEFAULT.KYLIN_CAL_DT--learn_kylin.json | 4 +-
...AULT.KYLIN_CATEGORY_GROUPINGS--learn_kylin.json | 4 +-
.../table/DEFAULT.KYLIN_COUNTRY--learn_kylin.json | 4 +-
.../table/DEFAULT.KYLIN_SALES--learn_kylin.json | 4 +-
.../apache/kylin/rest/job/MetadataCleanupJob.java | 161 ++++++++++
.../kylin/rest/job/StorageCleanJobHbaseUtil.java | 107 ++++---
.../apache/kylin/rest/job/StorageCleanupJob.java | 349 +++++++++++++--------
.../kylin/rest/job}/MetadataCleanupJobTest.java | 2 +-
.../kylin/rest/job/StorageCleanupJobTest.java | 7 +-
.../src/test/resources/test_meta/UUID | Bin
.../test_meta/cube/ci_inner_join_cube.json | 0
.../test_meta/cube_desc/ci_inner_join_cube.json | 0
.../04b6db34-b243-4780-855a-4c7ea4cee817.seq | Bin
.../NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict | Bin
.../NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict | Bin
.../execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930 | 0
.../d861b8b7-c773-47ab-bb1e-c8782ae8d930 | 0
.../d861b8b7-c773-47ab-bb1e-c8782ae8d930-00 | 0
.../d861b8b7-c773-47ab-bb1e-c8782ae8d930-01 | 0
.../src/test/resources/test_meta/kylin.properties | 0
.../test_meta/model_desc/ci_inner_join_model.json | 0
.../test/resources/test_meta/project/default.json | 0
.../test_meta/table/DEFAULT.TEST_ACCOUNT.json | 0
.../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 0
.../test_meta/table/DEFAULT.TEST_COUNTRY.json | 0
.../test_meta/table/DEFAULT.TEST_KYLIN_FACT.json | 0
.../test_meta/table/DEFAULT.TEST_ORDER.json | 0
.../resources/test_meta/table/EDW.TEST_CAL_DT.json | 0
.../test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json | 0
.../resources/test_meta/table/EDW.TEST_SITES.json | 0
.../92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot | Bin
.../7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot | Bin
.../kylin/rest/service/AdminServiceTest.java | 58 ++--
.../kylin/rest/service/QueryServiceTest.java | 28 +-
.../org/apache/kylin/source/hive/HiveMRInput.java | 24 +-
.../apache/kylin/source/hive/HiveMRInputTest.java | 25 +-
.../v2/coprocessor/endpoint/CubeVisitService.java | 12 +-
.../org/apache/kylin/tool/MetadataCleanupJob.java | 155 ++-------
57 files changed, 707 insertions(+), 439 deletions(-)
copy build/bin/{get-properties.sh => check-hive-usability.sh} (68%)
mode change 100755 => 100644
create mode 100644 server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
rename {tool/src/test/java/org/apache/kylin/tool => server-base/src/test/java/org/apache/kylin/rest/job}/MetadataCleanupJobTest.java (98%)
rename {tool => server-base}/src/test/resources/test_meta/UUID (100%)
rename {tool => server-base}/src/test/resources/test_meta/cube/ci_inner_join_cube.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/cube_desc/ci_inner_join_cube.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/cube_statistics/kylin_sales_cube/04b6db34-b243-4780-855a-4c7ea4cee817.seq (100%)
rename {tool => server-base}/src/test/resources/test_meta/dict/DEFAULT.KYLIN_COUNTRY/NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict (100%)
rename {tool => server-base}/src/test/resources/test_meta/dict/DEFAULT.TEST_COUNTRY/NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict (100%)
rename {tool => server-base}/src/test/resources/test_meta/execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930 (100%)
rename {tool => server-base}/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930 (100%)
rename {tool => server-base}/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-00 (100%)
rename {tool => server-base}/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-01 (100%)
rename {tool => server-base}/src/test/resources/test_meta/kylin.properties (100%)
rename {tool => server-base}/src/test/resources/test_meta/model_desc/ci_inner_join_model.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/project/default.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/DEFAULT.TEST_ACCOUNT.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/DEFAULT.TEST_COUNTRY.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/DEFAULT.TEST_KYLIN_FACT.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/DEFAULT.TEST_ORDER.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/EDW.TEST_CAL_DT.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table/EDW.TEST_SITES.json (100%)
rename {tool => server-base}/src/test/resources/test_meta/table_snapshot/DEFAULT.KYLIN_COUNTRY/92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot (100%)
rename {tool => server-base}/src/test/resources/test_meta/table_snapshot/DEFAULT.TEST_COUNTRY/7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot (100%)
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 06/07: Minor, add config for enable and disable auto merge
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 143cfdc317c4faa2289cdd84bfb3685cc17dc426
Author: nichunen <ch...@kyligence.io>
AuthorDate: Mon Apr 16 09:41:57 2018 +0800
Minor, add config for enable and disable auto merge
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 458fe8d..a971f14 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -483,6 +483,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.gtscanrequest-serialization-level", "1"));
}
+ public boolean isAutoMergeEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.is-automerge-enabled", "true"));
+ }
+
// ============================================================================
// Cube Planner
// ============================================================================
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index ea5006e..035cf7b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -574,7 +574,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
return false;
- return this.getDescriptor().getAutoMergeTimeRanges() != null
+ return this.getConfig().isAutoMergeEnabled() && this.getDescriptor().getAutoMergeTimeRanges() != null
&& this.getDescriptor().getAutoMergeTimeRanges().length > 0;
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 07/07: KYLIN-3340, enhance hql check
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b1c8944471558afe63d61b6578c494decbae53cc
Author: Cheng Wang <ch...@kyligence.io>
AuthorDate: Mon Apr 16 15:17:29 2018 +0800
KYLIN-3340, enhance hql check
---
.../org/apache/kylin/source/hive/HiveMRInput.java | 24 +++++++++++++---------
.../apache/kylin/source/hive/HiveMRInputTest.java | 19 +++++++++++++++++
2 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 38656cf..a96f4d5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -238,16 +238,8 @@ public class HiveMRInput implements IMRInput {
String identity = lookUpTableDesc.getIdentity();
String intermediate = lookUpTableDesc.getMaterializedName();
if (lookUpTableDesc.isView()) {
- StringBuilder createIntermediateTableHql = new StringBuilder();
- createIntermediateTableHql.append("DROP TABLE IF EXISTS " + intermediate + ";\n");
- createIntermediateTableHql
- .append("CREATE EXTERNAL TABLE IF NOT EXISTS " + intermediate + " LIKE " + identity + "\n");
- createIntermediateTableHql.append("LOCATION '" + jobWorkingDir + "/" + intermediate + "';\n");
- createIntermediateTableHql
- .append("ALTER TABLE " + intermediate + " SET TBLPROPERTIES('auto.purge'='true');\n");
- createIntermediateTableHql
- .append("INSERT OVERWRITE TABLE " + intermediate + " SELECT * FROM " + identity + ";\n");
- hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
+ String materializeViewHql = materializeViewHql(intermediate, identity, jobWorkingDir);
+ hiveCmdBuilder.addStatement(materializeViewHql);
hiveViewIntermediateTables = hiveViewIntermediateTables + intermediate + ";";
}
}
@@ -259,6 +251,18 @@ public class HiveMRInput implements IMRInput {
return step;
}
+ // each append must be a complete hql.
+ public static String materializeViewHql(String viewName, String tableName, String jobWorkingDir) {
+ StringBuilder createIntermediateTableHql = new StringBuilder();
+ createIntermediateTableHql.append("DROP TABLE IF EXISTS " + viewName + ";\n");
+ createIntermediateTableHql.append("CREATE EXTERNAL TABLE IF NOT EXISTS " + viewName + " LIKE " + tableName
+ + " LOCATION '" + jobWorkingDir + "/" + viewName + "';\n");
+ createIntermediateTableHql.append("ALTER TABLE " + viewName + " SET TBLPROPERTIES('auto.purge'='true');\n");
+ createIntermediateTableHql
+ .append("INSERT OVERWRITE TABLE " + viewName + " SELECT * FROM " + tableName + ";\n");
+ return createIntermediateTableHql.toString();
+ }
+
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 8df3174..a81cc21 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -56,4 +56,23 @@ public class HiveMRInputTest {
}
}
+ @Test
+ public void testMaterializeViewHql() {
+ final int viewSize = 2;
+ String[] mockedViewNames = { "mockedView1", "mockedView2" };
+ String[] mockedTalbeNames = { "mockedTable1", "mockedTable2" };
+ String mockedWorkingDir = "mockedWorkingDir";
+
+ StringBuilder hqls = new StringBuilder();
+ for (int i = 0; i < viewSize; i++) {
+ String hql = HiveMRInput.BatchCubingInputSide.materializeViewHql(mockedViewNames[i], mockedTalbeNames[i],
+ mockedWorkingDir);
+ hqls.append(hql);
+ }
+
+ for (String sub : hqls.toString().split("\n")) {
+ Assert.assertTrue(sub.endsWith(";"));
+ }
+ }
+
}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 01/07: KYLIN-3331 Add script check-hive-usability.sh to
avoid script hangs
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6fb75b68c222b99c39d6be905ce9b6e42b286d3d
Author: nichunen <ch...@kyligence.io>
AuthorDate: Sun Apr 8 21:25:26 2018 +0800
KYLIN-3331 Add script check-hive-usability.sh to avoid script hangs
---
build/bin/check-hive-usability.sh | 45 +++++++++++++++++++++++++++++++++++++++
build/bin/find-hive-dependency.sh | 1 +
2 files changed, 46 insertions(+)
diff --git a/build/bin/check-hive-usability.sh b/build/bin/check-hive-usability.sh
new file mode 100644
index 0000000..57bf3c4
--- /dev/null
+++ b/build/bin/check-hive-usability.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+hive -e "select 1" > /dev/null 2>&1 &
+pid=$!
+
+((timeLeft = 60))
+
+while ((timeLeft > 0)); do
+ sleep 5
+ ps -p $pid > /dev/null || break
+ ((timeLeft -= 5))
+done
+
+if ps -p $pid > /dev/null
+then
+ kill $pid
+ sleep 5
+
+ # still alive, use kill -9
+ if ps -p $pid > /dev/null
+ then
+ kill -9 $pid
+ fi
+
+ quit "ERROR: Check hive's usability failed, please check the status of your cluster"
+fi
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 2567d51..c647af1 100755
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -33,6 +33,7 @@ then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
hive_env=`${beeline_shell} ${hive_conf_properties} ${beeline_params} --outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' `
else
+ source ${dir}/check-hive-usability.sh
hive_env=`hive ${hive_conf_properties} -e set 2>&1 | grep 'env:CLASSPATH'`
fi
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 02/07: KYLIN-3328 Upgrade the metadata of sample cube to
latest
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9509989800170c49379b5442297e886df9df63ee
Author: nichunen <ch...@kyligence.io>
AuthorDate: Sun Apr 1 20:48:02 2018 +0800
KYLIN-3328 Upgrade the metadata of sample cube to latest
---
.../template/cube/kylin_sales_cube.json | 4 +--
.../template/cube/kylin_streaming_cube.json | 1 +
.../template/cube_desc/kylin_sales_cube.json | 14 +++++---
.../template/cube_desc/kylin_streaming_cube.json | 2 +-
.../template/model_desc/kylin_sales_model.json | 41 ++++++++++++++--------
.../sample_cube/template/project/learn_kylin.json | 4 ++-
.../table/DEFAULT.KYLIN_ACCOUNT--learn_kylin.json | 4 ++-
.../table/DEFAULT.KYLIN_CAL_DT--learn_kylin.json | 4 ++-
...AULT.KYLIN_CATEGORY_GROUPINGS--learn_kylin.json | 4 ++-
.../table/DEFAULT.KYLIN_COUNTRY--learn_kylin.json | 4 ++-
.../table/DEFAULT.KYLIN_SALES--learn_kylin.json | 4 ++-
11 files changed, 58 insertions(+), 28 deletions(-)
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json b/examples/sample_cube/template/cube/kylin_sales_cube.json
index 3d6d299..be07ffb 100644
--- a/examples/sample_cube/template/cube/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -5,8 +5,8 @@
"name" : "kylin_sales_cube",
"owner" : null,
"descriptor" : "kylin_sales_cube",
+ "display_name" : "kylin_sales_cube",
"status" : "DISABLED",
"segments" : [ ],
- "create_time" : null,
- "auto_merge_time_ranges" : [604800000, 2419200000]
+ "create_time_utc" : 0
}
\ No newline at end of file
diff --git a/examples/sample_cube/template/cube/kylin_streaming_cube.json b/examples/sample_cube/template/cube/kylin_streaming_cube.json
index 685fa59..e5286c3 100644
--- a/examples/sample_cube/template/cube/kylin_streaming_cube.json
+++ b/examples/sample_cube/template/cube/kylin_streaming_cube.json
@@ -7,6 +7,7 @@
"segments": [],
"last_modified": 0,
"descriptor": "kylin_streaming_cube",
+ "display_name" : "kylin_streaming_cube",
"size_kb": 0,
"input_records_count": 0,
"input_records_size": 0
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 1f16a5b..124fc73 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -3,8 +3,9 @@
"last_modified" : 1451468470824,
"version": "%default_version%",
"name" : "kylin_sales_cube",
+ "is_draft" : false,
"model_name" : "kylin_sales_model",
- "description" : null,
+ "description" : "",
"null_string" : null,
"dimensions" : [ {
"name" : "TRANS_ID",
@@ -264,14 +265,19 @@
[ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
}
} ],
- "notify_list" : null,
+ "notify_list" : [ ],
"status_need_notify" : [ ],
"partition_date_start" : 1325376000000,
- "auto_merge_time_ranges" : null,
+ "partition_date_end" : 3153600000000,
+ "auto_merge_time_ranges" : [ ],
+ "volatile_range" : 0,
"retention_range" : 0,
"engine_type" : %default_engine_type%,
"storage_type" : %default_storage_type%,
"override_kylin_properties" : {
"kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
- }
+ },
+ "cuboid_black_list" : [ ],
+ "parent_forward" : 3,
+ "mandatory_dimension_set_list" : [ ]
}
diff --git a/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json b/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
index 21466e0..92775e0 100644
--- a/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_streaming_cube.json
@@ -143,7 +143,7 @@
},
"notify_list" : [ ],
"status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
+ "auto_merge_time_ranges" : [ ],
"retention_range" : 0,
"engine_type" : %default_engine_type%,
"storage_type" : %default_storage_type%,
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index bcfc1ec..0a93949 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -1,20 +1,26 @@
{
"uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
"name" : "kylin_sales_model",
+ "is_draft" : false,
+ "description" : "",
"fact_table" : "DEFAULT.KYLIN_SALES",
"lookups" : [ {
"table" : "DEFAULT.KYLIN_CAL_DT",
+ "kind" : "LOOKUP",
+ "alias" : "KYLIN_CAL_DT",
"join" : {
"type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "PART_DT" ]
+ "primary_key" : [ "KYLIN_CAL_DT.CAL_DT" ],
+ "foreign_key" : [ "KYLIN_SALES.PART_DT" ]
}
}, {
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "kind" : "LOOKUP",
+ "alias" : "KYLIN_CATEGORY_GROUPINGS",
"join" : {
"type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
+ "primary_key" : [ "KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID", "KYLIN_CATEGORY_GROUPINGS.SITE_ID" ],
+ "foreign_key" : [ "KYLIN_SALES.LEAF_CATEG_ID", "KYLIN_SALES.LSTG_SITE_ID" ]
}
}, {
"table" : "DEFAULT.KYLIN_ACCOUNT",
@@ -22,8 +28,8 @@
"kind" : "LOOKUP",
"join" : {
"type" : "inner",
- "primary_key" : [ "ACCOUNT_ID" ],
- "foreign_key" : [ "BUYER_ID" ]
+ "primary_key" : [ "BUYER_ACCOUNT.ACCOUNT_ID" ],
+ "foreign_key" : [ "KYLIN_SALES.BUYER_ID" ]
}
}, {
"table" : "DEFAULT.KYLIN_ACCOUNT",
@@ -31,8 +37,8 @@
"kind" : "LOOKUP",
"join" : {
"type" : "inner",
- "primary_key" : [ "ACCOUNT_ID" ],
- "foreign_key" : [ "SELLER_ID" ]
+ "primary_key" : [ "SELLER_ACCOUNT.ACCOUNT_ID" ],
+ "foreign_key" : [ "KYLIN_SALES.SELLER_ID" ]
}
}, {
"table" : "DEFAULT.KYLIN_COUNTRY",
@@ -40,7 +46,7 @@
"kind" : "LOOKUP",
"join" : {
"type" : "inner",
- "primary_key" : [ "COUNTRY" ],
+ "primary_key" : [ "BUYER_COUNTRY.COUNTRY" ],
"foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ]
}
}, {
@@ -49,7 +55,7 @@
"kind" : "LOOKUP",
"join" : {
"type" : "inner",
- "primary_key" : [ "COUNTRY" ],
+ "primary_key" : [ "SELLER_COUNTRY.COUNTRY" ],
"foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ]
}
}],
@@ -76,14 +82,19 @@
"columns" : [ "COUNTRY", "NAME" ]
} ],
"metrics": [
- "PRICE",
- "ITEM_COUNT"
+ "KYLIN_SALES.PRICE",
+ "KYLIN_SALES.ITEM_COUNT"
],
"last_modified" : 1422435345362,
- "filter_condition" : null,
+ "filter_condition" : "",
"partition_desc" : {
"partition_date_column" : "KYLIN_SALES.PART_DT",
+ "partition_time_column" : null,
"partition_date_start" : 1325376000000,
- "partition_type" : "APPEND"
- }
+ "partition_date_format" : "yyyy-MM-dd",
+ "partition_time_format" : "HH:mm:ss",
+ "partition_type" : "APPEND",
+ "partition_condition_builder" : "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ },
+ "capacity" : "MEDIUM"
}
diff --git a/examples/sample_cube/template/project/learn_kylin.json b/examples/sample_cube/template/project/learn_kylin.json
index 1a74691..e5afa02 100644
--- a/examples/sample_cube/template/project/learn_kylin.json
+++ b/examples/sample_cube/template/project/learn_kylin.json
@@ -24,5 +24,7 @@
"models": [
"kylin_sales_model",
"kylin_streaming_model"
- ]
+ ],
+ "ext_filters": [],
+ "override_kylin_properties": {}
}
\ No newline at end of file
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT--learn_kylin.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT--learn_kylin.json
index f1b7755..fd5556a 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT--learn_kylin.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT--learn_kylin.json
@@ -24,5 +24,7 @@
"datatype" : "string"
} ],
"database" : "DEFAULT",
- "last_modified" : 0
+ "last_modified" : 0,
+ "source_type" : 0,
+ "table_type" : "MANAGED_TABLE"
}
\ No newline at end of file
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT--learn_kylin.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT--learn_kylin.json
index 1b46d04..e83bb00 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT--learn_kylin.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT--learn_kylin.json
@@ -404,5 +404,7 @@
"datatype" : "string"
} ],
"database" : "DEFAULT",
- "last_modified" : 0
+ "last_modified" : 0,
+ "source_type" : 0,
+ "table_type" : "MANAGED_TABLE"
}
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS--learn_kylin.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS--learn_kylin.json
index accba35..45e216f 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS--learn_kylin.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS--learn_kylin.json
@@ -148,5 +148,7 @@
"datatype" : "string"
} ],
"database" : "DEFAULT",
- "last_modified" : 0
+ "last_modified" : 0,
+ "source_type" : 0,
+ "table_type" : "MANAGED_TABLE"
}
\ No newline at end of file
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_COUNTRY--learn_kylin.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_COUNTRY--learn_kylin.json
index 8d38810..62c8276 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_COUNTRY--learn_kylin.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_COUNTRY--learn_kylin.json
@@ -20,5 +20,7 @@
"datatype" : "string"
} ],
"database" : "DEFAULT",
- "last_modified" : 0
+ "last_modified" : 0,
+ "source_type" : 0,
+ "table_type" : "MANAGED_TABLE"
}
\ No newline at end of file
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES--learn_kylin.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES--learn_kylin.json
index 9286215..caf4731 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES--learn_kylin.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES--learn_kylin.json
@@ -52,5 +52,7 @@
"datatype" : "string"
} ],
"database" : "DEFAULT",
- "last_modified" : 0
+ "last_modified" : 0,
+ "source_type" : 0,
+ "table_type" : "MANAGED_TABLE"
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 05/07: KYLIN-3337 Use an auto-closeable style API to
replace KylinConfig.setKylinConfigThreadLocal()
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a9dcad1a859158ffeecceff614ed864695788a0d
Author: Li Yang <li...@apache.org>
AuthorDate: Sat Apr 14 07:17:32 2018 +0800
KYLIN-3337 Use an auto-closeable style API to replace KylinConfig.setKylinConfigThreadLocal()
---
.../java/org/apache/kylin/common/KylinConfig.java | 29 ++++++++---
.../org/apache/kylin/common/KylinConfigTest.java | 32 +++++++-----
.../kylin/engine/mr/common/AbstractHadoopJob.java | 8 ++-
.../kylin/rest/service/AdminServiceTest.java | 58 ++++++++++++----------
.../kylin/rest/service/QueryServiceTest.java | 28 ++++++-----
.../apache/kylin/source/hive/HiveMRInputTest.java | 6 +--
.../v2/coprocessor/endpoint/CubeVisitService.java | 12 +++--
7 files changed, 103 insertions(+), 70 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 105cd9e..81c9b89 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -239,18 +239,31 @@ public class KylinConfig extends KylinConfigBase {
setKylinConfigInEnvIfMissing(props);
}
- public static void setKylinConfigThreadLocal(KylinConfig config) {
- THREAD_ENV_INSTANCE.set(config);
+ // auto-closeable API to remind that a thread local config must always be removed
+ public static SetAndUnsetThreadLocalConfig setAndUnsetThreadLocalConfig(KylinConfig config) {
+ return new SetAndUnsetThreadLocalConfig(config);
}
+
+ public static class SetAndUnsetThreadLocalConfig implements AutoCloseable {
+
+ public SetAndUnsetThreadLocalConfig(KylinConfig config) {
+ THREAD_ENV_INSTANCE.set(config);
+ }
- public static void removeKylinConfigThreadLocal() {
- THREAD_ENV_INSTANCE.remove();
+ @Override
+ public void close() {
+ THREAD_ENV_INSTANCE.remove();
+ }
}
- public static KylinConfig createKylinConfig(String propsInStr) throws IOException {
- Properties props = new Properties();
- props.load(new StringReader(propsInStr));
- return createKylinConfig(props);
+ public static KylinConfig createKylinConfig(String propsInStr) {
+ try {
+ Properties props = new Properties();
+ props.load(new StringReader(propsInStr));
+ return createKylinConfig(props);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to create KylinConfig from string: " + propsInStr, e);
+ }
}
public static KylinConfig createKylinConfig(KylinConfig another) {
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 92c730a..811c574 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertTrue;
import java.util.Map;
import java.util.Properties;
+import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.junit.Test;
import com.google.common.collect.Maps;
@@ -98,7 +99,7 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase {
}
@Test
- public void testThreadLocalOverride() {
+ public void testThreadLocalOverride() throws InterruptedException {
final String metadata1 = "meta1";
final String metadata2 = "meta2";
@@ -111,18 +112,23 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase {
// test thread-local override
KylinConfig threadConfig = KylinConfig.createKylinConfig(new Properties());
threadConfig.setMetadataUrl(metadata2);
- KylinConfig.setKylinConfigThreadLocal(threadConfig);
-
- assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString());
-
- // other threads still use system KylinConfig
- new Thread(new Runnable() {
- @Override
- public void run() {
- System.out.println("Started new thread.");
- assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString());
- }
- }).start();
+
+ try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(threadConfig)) {
+
+ assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString());
+
+ // other threads still use system KylinConfig
+ final String[] holder = new String[1];
+ Thread child = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ holder[0] = KylinConfig.getInstanceFromEnv().getMetadataUrl().toString();
+ }
+ });
+ child.start();
+ child.join();
+ assertEquals(metadata1, holder[0]);
+ }
}
@Test
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 8925a8e..7b25354 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.apache.kylin.common.StorageURL;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
@@ -495,7 +496,12 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
} catch (IOException e) {
throw new RuntimeException(e);
}
- KylinConfig.setKylinConfigThreadLocal(config);
+
+ // This is a bad example where the thread local KylinConfig cannot be auto-closed due to
+ // limitation of MR API. It works because MR task runs its own process. Do not copy.
+ @SuppressWarnings("unused")
+ SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
return config;
}
diff --git a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
index 8d49abc..cc0421a 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -44,33 +45,36 @@ public class AdminServiceTest extends ServiceTestBase {
FileUtils.deleteQuietly(file);
FileUtils.touch(file);
String path = Thread.currentThread().getContextClassLoader().getResource("kylin.properties").getPath();
- KylinConfig.setKylinConfigThreadLocal(KylinConfig.createInstanceFromUri(path));
- String expected = "kylin.web.link-streaming-guide=http://kylin.apache.org/\n" +
- "kylin.web.dashboard-enabled=\n" +
- "kylin.web.contact-mail=\n" +
- "kylin.query.cache-enabled=true\n" +
- "kylin.web.link-diagnostic=\n" +
- "kylin.web.help.length=4\n" +
- "kylin.web.timezone=GMT+8\n" +
- "kylin.server.external-acl-provider=\n" +
- "kylin.tool.auto-migrate-cube.enabled=\n" +
- "kylin.storage.default=2\n" +
- "kylin.cube.cubeplanner.enabled=false\n" +
- "kylin.web.help=\n" +
- "kylin.web.export-allow-other=true\n" +
- "kylin.web.link-hadoop=\n" +
- "kylin.web.hide-measures=RAW\n" +
- "kylin.htrace.show-gui-trace-toggle=false\n" +
- "kylin.web.export-allow-admin=true\n" +
- "kylin.env=QA\n" +
- "kylin.web.hive-limit=20\n" +
- "kylin.engine.default=2\n" +
- "kylin.web.help.3=onboard|Cube Design Tutorial|http://kylin.apache.org/docs21/howto/howto_optimize_cubes.html\n" +
- "kylin.web.help.2=tableau|Tableau Guide|http://kylin.apache.org/docs21/tutorial/tableau_91.html\n" +
- "kylin.web.help.1=odbc|ODBC Driver|http://kylin.apache.org/docs21/tutorial/odbc.html\n" +
- "kylin.web.help.0=start|Getting Started|http://kylin.apache.org/docs21/tutorial/kylin_sample.html\n" +
- "kylin.security.profile=testing\n";
- Assert.assertEquals(expected, adminService.getPublicConfig());
+ KylinConfig config = KylinConfig.createInstanceFromUri(path);
+ try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
+
+ String expected = "kylin.web.link-streaming-guide=http://kylin.apache.org/\n" +
+ "kylin.web.dashboard-enabled=\n" +
+ "kylin.web.contact-mail=\n" +
+ "kylin.query.cache-enabled=true\n" +
+ "kylin.web.link-diagnostic=\n" +
+ "kylin.web.help.length=4\n" +
+ "kylin.web.timezone=GMT+8\n" +
+ "kylin.server.external-acl-provider=\n" +
+ "kylin.tool.auto-migrate-cube.enabled=\n" +
+ "kylin.storage.default=2\n" +
+ "kylin.cube.cubeplanner.enabled=false\n" +
+ "kylin.web.help=\n" +
+ "kylin.web.export-allow-other=true\n" +
+ "kylin.web.link-hadoop=\n" +
+ "kylin.web.hide-measures=RAW\n" +
+ "kylin.htrace.show-gui-trace-toggle=false\n" +
+ "kylin.web.export-allow-admin=true\n" +
+ "kylin.env=QA\n" +
+ "kylin.web.hive-limit=20\n" +
+ "kylin.engine.default=2\n" +
+ "kylin.web.help.3=onboard|Cube Design Tutorial|http://kylin.apache.org/docs21/howto/howto_optimize_cubes.html\n" +
+ "kylin.web.help.2=tableau|Tableau Guide|http://kylin.apache.org/docs21/tutorial/tableau_91.html\n" +
+ "kylin.web.help.1=odbc|ODBC Driver|http://kylin.apache.org/docs21/tutorial/odbc.html\n" +
+ "kylin.web.help.0=start|Getting Started|http://kylin.apache.org/docs21/tutorial/kylin_sample.html\n" +
+ "kylin.security.profile=testing\n";
+ Assert.assertEquals(expected, adminService.getPublicConfig());
+ }
}
}
diff --git a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 5c633a3..f3dc6b1 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -21,8 +21,9 @@ package org.apache.kylin.rest.service;
import java.io.IOException;
import java.sql.SQLException;
-import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
+import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryContextFacade;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -79,21 +80,22 @@ public class QueryServiceTest extends ServiceTestBase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
config.setProperty("kylin.query.convert-create-table-to-with", "true");
- KylinConfig.setKylinConfigThreadLocal(config);
+ try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
- SQLRequest request = new SQLRequest();
- request.setProject("default");
- request.setSql(create_table1);
- queryService.doQueryWithCache(request);
+ SQLRequest request = new SQLRequest();
+ request.setProject("default");
+ request.setSql(create_table1);
+ queryService.doQueryWithCache(request);
- request.setSql(create_table2);
- queryService.doQueryWithCache(request);
+ request.setSql(create_table2);
+ queryService.doQueryWithCache(request);
- request.setSql(select_table);
- SQLResponse response = queryService.doQueryWithCache(request, true);
+ request.setSql(select_table);
+ SQLResponse response = queryService.doQueryWithCache(request, true);
- Assert.assertEquals(
- "WITH tableId as (select * from some_table1) , tableId2 AS (select * FROM some_table2) select * from tableId join tableId2 on tableId.a = tableId2.b;",
- response.getExceptionMessage());
+ Assert.assertEquals(
+ "WITH tableId as (select * from some_table1) , tableId2 AS (select * FROM some_table2) select * from tableId join tableId2 on tableId.a = tableId2.b;",
+ response.getExceptionMessage());
+ }
}
}
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index f33ee74..8df3174 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.junit.Assert;
import org.junit.Test;
@@ -38,9 +39,8 @@ public class HiveMRInputTest {
public void TestGetJobWorkingDir() throws IOException {
FileSystem fileSystem = FileSystem.get(new Configuration());
Path jobWorkDirPath = null;
- try {
- KylinConfig kylinConfig = mock(KylinConfig.class);
- KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ KylinConfig kylinConfig = mock(KylinConfig.class);
+ try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) {
when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index b784ab9..fd54e2b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
import org.apache.kylin.common.util.Bytes;
@@ -239,18 +240,19 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
CubeVisitProtos.CubeVisitResponse.ErrorInfo errorInfo = null;
+ // if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
+ KylinConfig kylinConfig = KylinConfig.createKylinConfig(request.getKylinProperties());
+
String queryId = request.hasQueryId() ? request.getQueryId() : "UnknownId";
logger.info("start query {} in thread {}", queryId, Thread.currentThread().getName());
- try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
+ try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig);
+ SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
+
final long serviceStartTime = System.currentTimeMillis();
region = (HRegion) env.getRegion();
region.startRegionOperation();
- // if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
- KylinConfig kylinConfig = KylinConfig.createKylinConfig(request.getKylinProperties());
- KylinConfig.setKylinConfigThreadLocal(kylinConfig);
-
debugGitTag = region.getTableDesc().getValue(IRealizationConstants.HTableGitTag);
final GTScanRequest scanReq = GTScanRequest.serializer
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 04/07: KYLIN-3289 Refactor the storage garbage clean up code
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 71397fd39709ebdb61b20309c22ef1ae8e3c2af4
Author: Li Yang <li...@apache.org>
AuthorDate: Fri Apr 13 13:16:02 2018 +0800
KYLIN-3289 Refactor the storage garbage clean up code
---
.../java/org/apache/kylin/common/KylinConfig.java | 2 +
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/common/util/AbstractApplication.java | 2 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 4 +
.../java/org/apache/kylin/common/util/Pair.java | 8 +
.../apache/kylin/rest/job/MetadataCleanupJob.java | 161 ++++++++++
.../kylin/rest/job/StorageCleanJobHbaseUtil.java | 107 ++++---
.../apache/kylin/rest/job/StorageCleanupJob.java | 349 +++++++++++++--------
.../kylin/rest/job}/MetadataCleanupJobTest.java | 2 +-
.../kylin/rest/job/StorageCleanupJobTest.java | 7 +-
.../src/test/resources/test_meta/UUID | Bin
.../test_meta/cube/ci_inner_join_cube.json | 0
.../test_meta/cube_desc/ci_inner_join_cube.json | 0
.../04b6db34-b243-4780-855a-4c7ea4cee817.seq | Bin
.../NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict | Bin
.../NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict | Bin
.../execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930 | 0
.../d861b8b7-c773-47ab-bb1e-c8782ae8d930 | 0
.../d861b8b7-c773-47ab-bb1e-c8782ae8d930-00 | 0
.../d861b8b7-c773-47ab-bb1e-c8782ae8d930-01 | 0
.../src/test/resources/test_meta/kylin.properties | 0
.../test_meta/model_desc/ci_inner_join_model.json | 0
.../test/resources/test_meta/project/default.json | 0
.../test_meta/table/DEFAULT.TEST_ACCOUNT.json | 0
.../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 0
.../test_meta/table/DEFAULT.TEST_COUNTRY.json | 0
.../test_meta/table/DEFAULT.TEST_KYLIN_FACT.json | 0
.../test_meta/table/DEFAULT.TEST_ORDER.json | 0
.../resources/test_meta/table/EDW.TEST_CAL_DT.json | 0
.../test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json | 0
.../resources/test_meta/table/EDW.TEST_SITES.json | 0
.../92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot | Bin
.../7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot | Bin
.../org/apache/kylin/tool/MetadataCleanupJob.java | 155 ++-------
34 files changed, 481 insertions(+), 318 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 191d767..105cd9e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -412,6 +412,8 @@ public class KylinConfig extends KylinConfigBase {
return (T) mgr;
try {
+ logger.info("Creating new manager instance of " + clz);
+
// new manager via static Manager.newInstance()
Method method = clz.getDeclaredMethod("newInstance", KylinConfig.class);
method.setAccessible(true); // override accessibility
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f558f42..458fe8d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -306,7 +306,7 @@ abstract public class KylinConfigBase implements Serializable {
public String getCacheSyncErrorHandler() {
return getOptional("kylin.metadata.sync-error-handler");
}
-
+
// for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java b/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
index 3909a3c..ab1e691 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
@@ -31,7 +31,7 @@ public abstract class AbstractApplication {
public final void execute(String[] args) {
OptionsHelper optionsHelper = new OptionsHelper();
- System.out.println("Abstract Application args:" + StringUtils.join(args, " "));
+ System.out.println("Running " + this.getClass().getName() + " " + StringUtils.join(args, " "));
try {
optionsHelper.parseOptions(getOptions(), args);
execute(optionsHelper);
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 867bc83..ed6ef89 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -81,6 +81,10 @@ public class HadoopUtil {
return getFileSystem(new Path(makeURI(path)));
}
+ public static FileSystem getFileSystem(String path, Configuration conf) throws IOException {
+ return getFileSystem(new Path(makeURI(path)), conf);
+ }
+
public static FileSystem getFileSystem(Path path) throws IOException {
Configuration conf = getCurrentConfiguration();
return getFileSystem(path, conf);
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Pair.java b/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
index aa85855..1cacf39 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
@@ -70,6 +70,10 @@ public class Pair<T1, T2> implements Serializable {
public T1 getFirst() {
return first;
}
+
+ public T1 getKey() {
+ return first;
+ }
/**
* Replace the first element of the pair.
@@ -86,6 +90,10 @@ public class Pair<T1, T2> implements Serializable {
public T2 getSecond() {
return second;
}
+
+ public T2 getValue() {
+ return second;
+ }
/**
* Replace the second element of the pair.
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
new file mode 100644
index 0000000..1aec429
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.job;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.dao.ExecutableDao;
+import org.apache.kylin.job.dao.ExecutablePO;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public class MetadataCleanupJob {
+
+ private static final Logger logger = LoggerFactory.getLogger(MetadataCleanupJob.class);
+
+ private static final long NEW_RESOURCE_THREADSHOLD_MS = 12 * 3600 * 1000L; // 12 hour
+
+ // ============================================================================
+
+ final KylinConfig config;
+
+ private List<String> garbageResources = Collections.emptyList();
+
+ public MetadataCleanupJob() {
+ this(KylinConfig.getInstanceFromEnv());
+ }
+
+ public MetadataCleanupJob(KylinConfig config) {
+ this.config = config;
+ }
+
+ public List<String> getGarbageResources() {
+ return garbageResources;
+ }
+
+ // function entrance
+ public List<String> cleanup(boolean delete, int jobOutdatedDays) throws Exception {
+ CubeManager cubeManager = CubeManager.getInstance(config);
+ ResourceStore store = ResourceStore.getStore(config);
+ long newResourceTimeCut = System.currentTimeMillis() - NEW_RESOURCE_THREADSHOLD_MS;
+
+ List<String> toDeleteCandidates = Lists.newArrayList();
+
+ // two level resources, snapshot tables and cube statistics
+ for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT,
+ ResourceStore.CUBE_STATISTICS_ROOT }) {
+ for (String dir : noNull(store.listResources(resourceRoot))) {
+ for (String res : noNull(store.listResources(dir))) {
+ if (store.getResourceTimestamp(res) < newResourceTimeCut)
+ toDeleteCandidates.add(res);
+ }
+ }
+ }
+
+ // three level resources, only dictionaries
+ for (String resourceRoot : new String[] { ResourceStore.DICT_RESOURCE_ROOT }) {
+ for (String dir : noNull(store.listResources(resourceRoot))) {
+ for (String dir2 : noNull(store.listResources(dir))) {
+ for (String res : noNull(store.listResources(dir2))) {
+ if (store.getResourceTimestamp(res) < newResourceTimeCut)
+ toDeleteCandidates.add(res);
+ }
+ }
+ }
+ }
+
+ // exclude resources in use
+ Set<String> activeResources = Sets.newHashSet();
+ for (CubeInstance cube : cubeManager.listAllCubes()) {
+ for (CubeSegment segment : cube.getSegments()) {
+ activeResources.addAll(segment.getSnapshotPaths());
+ activeResources.addAll(segment.getDictionaryPaths());
+ activeResources.add(segment.getStatisticsResourcePath());
+ }
+ }
+ toDeleteCandidates.removeAll(activeResources);
+
+ // delete old and completed jobs
+ long outdatedJobTimeCut = System.currentTimeMillis() - jobOutdatedDays * 24 * 3600 * 1000L;
+ ExecutableDao executableDao = ExecutableDao.getInstance(config);
+ List<ExecutablePO> allExecutable = executableDao.getJobs();
+ for (ExecutablePO executable : allExecutable) {
+ long lastModified = executable.getLastModified();
+ String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
+
+ if (lastModified < outdatedJobTimeCut && (ExecutableState.SUCCEED.toString().equals(jobStatus)
+ || ExecutableState.DISCARDED.toString().equals(jobStatus))) {
+ toDeleteCandidates.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
+ toDeleteCandidates.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());
+
+ for (ExecutablePO task : executable.getTasks()) {
+ toDeleteCandidates.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
+ }
+ }
+ }
+
+ garbageResources = cleanupConclude(delete, toDeleteCandidates);
+ return garbageResources;
+ }
+
+ private List<String> cleanupConclude(boolean delete, List<String> toDeleteResources) {
+ if (toDeleteResources.isEmpty()) {
+ logger.info("No metadata resource to clean up");
+ return toDeleteResources;
+ }
+
+ logger.info(toDeleteResources.size() + " metadata resource to clean up");
+
+ if (delete) {
+ ResourceStore store = ResourceStore.getStore(config);
+ for (String res : toDeleteResources) {
+ logger.info("Deleting metadata " + res);
+ try {
+ store.deleteResource(res);
+ } catch (IOException e) {
+ logger.error("Failed to delete resource " + res, e);
+ }
+ }
+ } else {
+ for (String res : toDeleteResources) {
+ logger.info("Dry run, pending delete metadata " + res);
+ }
+ }
+ return toDeleteResources;
+ }
+
+ private NavigableSet<String> noNull(NavigableSet<String> list) {
+ return (list == null) ? new TreeSet<String>() : list;
+ }
+
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index 61351ae..126c598 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -43,68 +43,76 @@ public class StorageCleanJobHbaseUtil {
protected static final Logger logger = LoggerFactory.getLogger(StorageCleanJobHbaseUtil.class);
- public static void cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {
+ @SuppressWarnings("deprecation")
+ public static List<String> cleanUnusedHBaseTables(boolean delete, int deleteTimeout) throws IOException {
try (HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create())) {
- cleanUnusedHBaseTables(hbaseAdmin, delete, deleteTimeout);
+ return cleanUnusedHBaseTables(hbaseAdmin, delete, deleteTimeout);
}
}
- static void cleanUnusedHBaseTables(HBaseAdmin hbaseAdmin, boolean delete, int deleteTimeout) throws IOException {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- CubeManager cubeMgr = CubeManager.getInstance(kylinConfig);
+ static List<String> cleanUnusedHBaseTables(HBaseAdmin hbaseAdmin, boolean delete, int deleteTimeout) throws IOException {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ CubeManager cubeMgr = CubeManager.getInstance(config);
+
// get all kylin hbase tables
- try {
- String namespace = kylinConfig.getHBaseStorageNameSpace();
- String tableNamePrefix = (namespace.equals("default") || namespace.equals(""))
- ? kylinConfig.getHBaseTableNamePrefix() : (namespace + ":" + kylinConfig.getHBaseTableNamePrefix());
- HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
- List<String> allTablesNeedToBeDropped = new ArrayList<String>();
- for (HTableDescriptor desc : tableDescriptors) {
- String host = desc.getValue(IRealizationConstants.HTableTag);
- if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
- //only take care htables that belongs to self, and created more than 2 days
- allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
- }
+ String namespace = config.getHBaseStorageNameSpace();
+ String tableNamePrefix = (namespace.equals("default") || namespace.equals(""))
+ ? config.getHBaseTableNamePrefix()
+ : (namespace + ":" + config.getHBaseTableNamePrefix());
+ HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
+ List<String> allTablesNeedToBeDropped = new ArrayList<String>();
+ for (HTableDescriptor desc : tableDescriptors) {
+ String host = desc.getValue(IRealizationConstants.HTableTag);
+ if (config.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
+ //only take care htables that belongs to self, and created more than 2 days
+ allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
}
+ }
- // remove every segment htable from drop list
- for (CubeInstance cube : cubeMgr.listAllCubes()) {
- for (CubeSegment seg : cube.getSegments()) {
- String tablename = seg.getStorageLocationIdentifier();
- if (allTablesNeedToBeDropped.contains(tablename)) {
- allTablesNeedToBeDropped.remove(tablename);
- logger.info("Exclude table " + tablename + " from drop list, as the table belongs to cube " + cube.getName() + " with status " + cube.getStatus());
- }
+ // remove every segment htable from drop list
+ for (CubeInstance cube : cubeMgr.listAllCubes()) {
+ for (CubeSegment seg : cube.getSegments()) {
+ String tablename = seg.getStorageLocationIdentifier();
+ if (allTablesNeedToBeDropped.contains(tablename)) {
+ allTablesNeedToBeDropped.remove(tablename);
+ logger.info("Exclude table " + tablename + " from drop list, as the table belongs to cube "
+ + cube.getName() + " with status " + cube.getStatus());
}
}
+ }
+
+ if (allTablesNeedToBeDropped.isEmpty()) {
+ logger.info("No HTable to clean up");
+ return allTablesNeedToBeDropped;
+ }
+
+ logger.info(allTablesNeedToBeDropped.size() + " HTable(s) to clean up");
- if (delete == true) {
- // drop tables
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- for (String htableName : allTablesNeedToBeDropped) {
- FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin, htableName));
- executorService.execute(futureTask);
- try {
- futureTask.get(deleteTimeout, TimeUnit.MINUTES);
- } catch (TimeoutException e) {
- logger.warn("It fails to delete htable " + htableName + ", for it cost more than " + deleteTimeout + " minutes!");
- futureTask.cancel(true);
- } catch (Exception e) {
- e.printStackTrace();
- futureTask.cancel(true);
- }
+ if (delete) {
+ // drop tables
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ for (String htableName : allTablesNeedToBeDropped) {
+ FutureTask futureTask = new FutureTask(new DeleteHTableRunnable(hbaseAdmin, htableName));
+ executorService.execute(futureTask);
+ try {
+ futureTask.get(deleteTimeout, TimeUnit.MINUTES);
+ } catch (TimeoutException e) {
+ logger.error("It fails to delete HTable " + htableName + ", for it cost more than "
+ + deleteTimeout + " minutes!", e);
+ futureTask.cancel(true);
+ } catch (Exception e) {
+ logger.error("Failed to delete HTable " + htableName, e);
+ futureTask.cancel(true);
}
- executorService.shutdown();
- } else {
- System.out.println("--------------- Tables To Be Dropped ---------------");
- for (String htableName : allTablesNeedToBeDropped) {
- System.out.println(htableName);
- }
- System.out.println("----------------------------------------------------");
}
- } catch (IOException e) {
- throw new IOException(e);
+ executorService.shutdown();
+ } else {
+ for (String htableName : allTablesNeedToBeDropped) {
+ logger.info("Dry run, pending delete HTable " + htableName);
+ }
}
+
+ return allTablesNeedToBeDropped;
}
static class DeleteHTableRunnable implements Callable {
@@ -131,4 +139,5 @@ public class StorageCleanJobHbaseUtil {
return null;
}
}
+
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 114050c..a7dfa01 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -22,6 +22,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
@@ -32,6 +34,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,6 +45,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -65,45 +69,61 @@ public class StorageCleanupJob extends AbstractApplication {
@SuppressWarnings("static-access")
protected static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false)
.withDescription("Delete the unused storage").create("delete");
+ @SuppressWarnings("static-access")
protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false)
.withDescription("Warning: will delete all kylin intermediate hive tables").create("force");
protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class);
- public static final int deleteTimeout = 10; // Unit minute
- protected FileSystem hbaseFs;
- protected FileSystem defaultFs;
+
+ // ============================================================================
+
+ final protected KylinConfig config;
+ final protected FileSystem hbaseFs;
+ final protected FileSystem defaultFs;
+ final protected ExecutableManager executableManager;
+
protected boolean delete = false;
protected boolean force = false;
- protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig
- .getInstanceFromEnv());
+
+ private List<String> hiveGarbageTables = Collections.emptyList();
+ private List<String> hbaseGarbageTables = Collections.emptyList();
+ private List<String> hdfsGarbageFiles = Collections.emptyList();
+ private long hdfsGarbageFileBytes = 0;
- public StorageCleanupJob(FileSystem defaultFs, FileSystem hbaseFs) {
+ public StorageCleanupJob() throws IOException {
+ this(KylinConfig.getInstanceFromEnv(), HadoopUtil.getWorkingFileSystem(),
+ HadoopUtil.getWorkingFileSystem(HBaseConfiguration.create()));
+ }
+
+ protected StorageCleanupJob(KylinConfig config, FileSystem defaultFs, FileSystem hbaseFs) {
+ this.config = config;
this.defaultFs = defaultFs;
this.hbaseFs = hbaseFs;
- }
-
- public StorageCleanupJob() throws IOException {
- this.defaultFs = HadoopUtil.getWorkingFileSystem();
- this.hbaseFs = HadoopUtil.getWorkingFileSystem(HBaseConfiguration.create());
+ this.executableManager = ExecutableManager.getInstance(config);
}
public void setDelete(boolean delete) {
this.delete = delete;
}
+
+ public void setForce(boolean force) {
+ this.force = force;
+ }
+
+ public List<String> getHiveGarbageTables() {
+ return hiveGarbageTables;
+ }
- protected void cleanUnusedHBaseTables() throws IOException {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- if ("hbase".equals(config.getMetadataUrl().getScheme())) {
- try {
- // use reflection to isolate NoClassDef errors when HBase is not available
- Class hbaseCleanUpUtil = Class.forName("org.apache.kylin.rest.job.StorageCleanJobHbaseUtil");
- Method cleanUnusedHBaseTables = hbaseCleanUpUtil.getDeclaredMethod("cleanUnusedHBaseTables",
- boolean.class, int.class);
- cleanUnusedHBaseTables.invoke(hbaseCleanUpUtil, delete, deleteTimeout);
- } catch (Throwable e) {
- throw new IOException(e);
- }
- }
+ public List<String> getHbaseGarbageTables() {
+ return hbaseGarbageTables;
+ }
+
+ public List<String> getHdfsGarbageFiles() {
+ return hdfsGarbageFiles;
+ }
+
+ public long getHdfsFileGarbageBytes() {
+ return hdfsGarbageFileBytes;
}
@Override
@@ -121,40 +141,98 @@ public class StorageCleanupJob extends AbstractApplication {
logger.info("force option value: '" + optionsHelper.getOptionValue(OPTION_FORCE) + "'");
delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
force = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_FORCE));
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- cleanUnusedIntermediateHiveTable(getHiveTables(), getCliCommandExecutor());
-
- if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) {
- cleanUnusedHdfsFiles(hbaseFs);
- }
- cleanUnusedHdfsFiles(defaultFs);
+
+ cleanup();
+ }
+
+ // function entrance
+ public void cleanup() throws Exception {
+
+ cleanUnusedIntermediateHiveTable();
cleanUnusedHBaseTables();
+ cleanUnusedHdfsFiles();
}
- protected List<String> getHiveTables() throws Exception {
- ISourceMetadataExplorer explr = SourceManager.getDefaultSource().getSourceMetadataExplorer();
- return explr.listTables(KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable());
+ protected void cleanUnusedHBaseTables() throws IOException {
+ if ("hbase".equals(config.getMetadataUrl().getScheme())) {
+ final int deleteTimeoutMin = 10; // Unit minute
+ try {
+ // use reflection to isolate NoClassDef errors when HBase is not available
+ Class hbaseCleanUpUtil = Class.forName("org.apache.kylin.rest.job.StorageCleanJobHbaseUtil");
+ Method cleanUnusedHBaseTables = hbaseCleanUpUtil.getDeclaredMethod("cleanUnusedHBaseTables",
+ boolean.class, int.class);
+ hbaseGarbageTables = (List<String>) cleanUnusedHBaseTables.invoke(hbaseCleanUpUtil, delete,
+ deleteTimeoutMin);
+ } catch (Throwable e) {
+ logger.error("Error during HBase clean up", e);
+ }
+ }
}
- protected CliCommandExecutor getCliCommandExecutor() throws IOException {
- return KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
+ protected class UnusedHdfsFileCollector {
+ LinkedHashSet<Pair<FileSystem, String>> list = new LinkedHashSet<>();
+
+ public void add(FileSystem fs, String path) {
+ list.add(Pair.newPair(fs, path));
+ }
}
+
+ private void cleanUnusedHdfsFiles() throws IOException {
+
+ UnusedHdfsFileCollector collector = new UnusedHdfsFileCollector();
+ collectUnusedHdfsFiles(collector);
+
+ if (collector.list.isEmpty()) {
+ logger.info("No HDFS files to clean up");
+ return;
+ }
+
+ long garbageBytes = 0;
+ List<String> garbageList = new ArrayList<>();
+
+ for (Pair<FileSystem, String> entry : collector.list) {
+ FileSystem fs = entry.getKey();
+ String path = entry.getValue();
- void cleanUnusedHdfsFiles(FileSystem fs) throws IOException {
- JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ try {
+ garbageList.add(path);
+ ContentSummary sum = fs.getContentSummary(new Path(path));
+ if (sum != null)
+ garbageBytes += sum.getLength();
+
+ if (delete) {
+ logger.info("Deleting HDFS path " + path);
+ fs.delete(new Path(path), true);
+ } else {
+ logger.info("Dry run, pending delete HDFS path " + path);
+ }
+ } catch (IOException e) {
+ logger.error("Error dealing unused HDFS path " + path, e);
+ }
+ }
+
+ hdfsGarbageFileBytes = garbageBytes;
+ hdfsGarbageFiles = garbageList;
+ }
+
+ protected void collectUnusedHdfsFiles(UnusedHdfsFileCollector collector) throws IOException {
+ if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) {
+ cleanUnusedHdfsFiles(hbaseFs, collector);
+ }
+ cleanUnusedHdfsFiles(defaultFs, collector);
+ }
+
+ private void cleanUnusedHdfsFiles(FileSystem fs, UnusedHdfsFileCollector collector) throws IOException {
+ final JobEngineConfig engineConfig = new JobEngineConfig(config);
+ final CubeManager cubeMgr = CubeManager.getInstance(config);
+
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
- // GlobFilter filter = new
- // GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
- // + "/kylin-.*");
- // TODO: when first use, /kylin/kylin_metadata does not exist.
+
try {
- FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
+ FileStatus[] fStatus = fs.listStatus(new Path(config.getHdfsWorkingDirectory()));
if (fStatus != null) {
for (FileStatus status : fStatus) {
String path = status.getPath().getName();
- // System.out.println(path);
if (path.startsWith("kylin-")) {
String kylinJobPath = engineConfig.getHdfsWorkingDirectory() + path;
allHdfsPathsNeedToBeDeleted.add(kylinJobPath);
@@ -162,12 +240,12 @@ public class StorageCleanupJob extends AbstractApplication {
}
}
} catch (FileNotFoundException e) {
- logger.info("Working Directory does not exist on HDFS. ");
+ logger.error("Working Directory does not exist on HDFS.", e);
}
+ // only remove FINISHED and DISCARDED job intermediate files
List<String> allJobs = executableManager.getAllJobIds();
for (String jobId : allJobs) {
- // only remove FINISHED and DISCARDED job intermediate files
final ExecutableState state = executableManager.getOutput(jobId).getState();
if (!state.isFinalState()) {
String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(), jobId);
@@ -189,35 +267,29 @@ public class StorageCleanupJob extends AbstractApplication {
}
}
}
+
+ // collect the garbage
+ for (String path : allHdfsPathsNeedToBeDeleted)
+ collector.add(fs, path);
+ }
- if (delete == true) {
- // remove files
- for (String hdfsPath : allHdfsPathsNeedToBeDeleted) {
- logger.info("Deleting hdfs path " + hdfsPath);
- Path p = new Path(hdfsPath);
- if (fs.exists(p) == true) {
- fs.delete(p, true);
- logger.info("Deleted hdfs path " + hdfsPath);
- } else {
- logger.info("Hdfs path " + hdfsPath + "does not exist");
- }
- }
- } else {
- System.out.println("--------------- HDFS Path To Be Deleted ---------------");
- for (String hdfsPath : allHdfsPathsNeedToBeDeleted) {
- System.out.println(hdfsPath);
- }
- System.out.println("-------------------------------------------------------");
+ private void cleanUnusedIntermediateHiveTable() throws Exception {
+ try {
+ cleanUnusedIntermediateHiveTableInternal();
+ } catch (NoClassDefFoundError e) {
+ if (e.getMessage().contains("HiveConf"))
+ logger.info("Skip cleanup of tntermediate Hive table, seems no Hive on classpath");
+ else
+ throw e;
}
}
-
- void cleanUnusedIntermediateHiveTable(List<String> hiveTableNames, CliCommandExecutor cmdExec) throws Exception {
- final KylinConfig config = KylinConfig.getInstanceFromEnv();
- JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
+
+ private void cleanUnusedIntermediateHiveTableInternal() throws Exception {
final int uuidLength = 36;
- final String preFix = MetadataConstants.KYLIN_INTERMEDIATE_PREFIX;
+ final String prefix = MetadataConstants.KYLIN_INTERMEDIATE_PREFIX;
final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
-
+
+ List<String> hiveTableNames = getHiveTables();
Iterable<String> kylinIntermediates = Iterables.filter(hiveTableNames, new Predicate<String>() {
@Override
public boolean apply(@Nullable String input) {
@@ -226,17 +298,14 @@ public class StorageCleanupJob extends AbstractApplication {
});
List<String> allJobs = executableManager.getAllJobIds();
- List<String> allHiveTablesNeedToBeDeleted = new ArrayList<String>();
List<String> workingJobList = new ArrayList<String>();
Map<String, String> segmentId2JobId = Maps.newHashMap();
- StringBuilder sb = new StringBuilder();
for (String jobId : allJobs) {
// only remove FINISHED and DISCARDED job intermediate table
final ExecutableState state = executableManager.getOutput(jobId).getState();
if (!state.isFinalState()) {
workingJobList.add(jobId);
- sb.append(jobId).append("(").append(state).append("), ");
}
try {
@@ -249,33 +318,35 @@ public class StorageCleanupJob extends AbstractApplication {
// some older version job metadata may fail to read, ignore it
}
}
- logger.info("Working jobIDs: " + workingJobList);
+ logger.debug("Working jobIDs: " + workingJobList);
- for (String line : kylinIntermediates) {
- logger.info("Checking table " + line);
+ // filter tables to delete
+ List<String> allHiveTablesNeedToBeDeleted = new ArrayList<String>();
+ for (String tableName : kylinIntermediates) {
+ logger.debug("Checking if table is garbage -- " + tableName);
- if (!line.startsWith(preFix))
+ if (!tableName.startsWith(prefix))
continue;
- if (force == true) {
- logger.warn("Warning: will delete all intermediate hive tables!!!!!!!!!!!!!!!!!!!!!!");
- allHiveTablesNeedToBeDeleted.add(line);
+ if (force) {
+ logger.debug("Force include table " + tableName);
+ allHiveTablesNeedToBeDeleted.add(tableName);
continue;
}
boolean isNeedDel = true;
- if (line.length() < preFix.length() + uuidLength) {
- logger.info("Skip deleting because length is not qualified");
+ if (tableName.length() < prefix.length() + uuidLength) {
+ logger.debug("Skip table because length is not qualified, " + tableName);
continue;
}
- String uuid = line.substring(line.length() - uuidLength, line.length());
+ String uuid = tableName.substring(tableName.length() - uuidLength, tableName.length());
uuid = uuid.replace("_", "-");
final Pattern UUID_PATTERN = Pattern.compile(uuidPattern);
if (!UUID_PATTERN.matcher(uuid).matches()) {
- logger.info("Skip deleting because pattern doesn't match");
+ logger.debug("Skip table because pattern doesn't match, " + tableName);
continue;
}
@@ -283,60 +354,82 @@ public class StorageCleanupJob extends AbstractApplication {
if (allJobs.contains(uuid)) {
isNeedDel = !workingJobList.contains(uuid);
} else if (isTableInUse(uuid, workingJobList)) {
- logger.info("Skip deleting because the table is in use");
+ logger.debug("Skip table because the table is in use, " + tableName);
isNeedDel = false;
}
if (isNeedDel) {
- allHiveTablesNeedToBeDeleted.add(line);
+ allHiveTablesNeedToBeDeleted.add(tableName);
}
}
- if (delete == true) {
-
+ // conclude hive tables to delete
+ hiveGarbageTables = allHiveTablesNeedToBeDeleted;
+ if (allHiveTablesNeedToBeDeleted.isEmpty()) {
+ logger.info("No Hive tables to clean up");
+ return;
+ }
+
+ if (delete) {
try {
- final String useDatabaseHql = "USE " + config.getHiveDatabaseForIntermediateTable() + ";";
- final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- hiveCmdBuilder.addStatement(useDatabaseHql);
- for (String delHive : allHiveTablesNeedToBeDeleted) {
- hiveCmdBuilder.addStatement("drop table if exists " + delHive + "; ");
- logger.info("Remove " + delHive + " from hive tables.");
- }
- cmdExec.execute(hiveCmdBuilder.build());
-
- //if kylin.source.hive.keep-flat-table, some intermediate table might be kept
- //delete external path
- for (String tableToDelete : allHiveTablesNeedToBeDeleted) {
- String uuid = tableToDelete.substring(tableToDelete.length() - uuidLength, tableToDelete.length());
- String segmentId = uuid.replace("_", "-");
-
- if (segmentId2JobId.containsKey(segmentId)) {
- String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(),
- segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
- Path externalDataPath = new Path(path);
- if (defaultFs.exists(externalDataPath)) {
- defaultFs.delete(externalDataPath, true);
- logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);
- } else {
- logger.info(
- "Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)",
- tableToDelete, path);
- }
- } else {
- logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete,
- segmentId2JobId.toString());
- }
- }
+ deleteHiveTables(allHiveTablesNeedToBeDeleted, segmentId2JobId);
} catch (IOException e) {
- e.printStackTrace();
+ logger.error("Error during deleting Hive tables", e);
}
-
} else {
- System.out.println("------ Intermediate Hive Tables To Be Dropped ------");
- for (String hiveTable : allHiveTablesNeedToBeDeleted) {
- System.out.println(hiveTable);
+ for (String table : allHiveTablesNeedToBeDeleted) {
+ logger.info("Dry run, pending delete Hive table " + table);
+ }
+ }
+ }
+
+ // override by test
+ protected List<String> getHiveTables() throws Exception {
+ ISourceMetadataExplorer explr = SourceManager.getDefaultSource().getSourceMetadataExplorer();
+ return explr.listTables(config.getHiveDatabaseForIntermediateTable());
+ }
+
+ // override by test
+ protected CliCommandExecutor getCliCommandExecutor() throws IOException {
+ return config.getCliCommandExecutor();
+ }
+
+ private void deleteHiveTables(List<String> allHiveTablesNeedToBeDeleted, Map<String, String> segmentId2JobId)
+ throws IOException {
+ final JobEngineConfig engineConfig = new JobEngineConfig(config);
+ final int uuidLength = 36;
+
+ final String useDatabaseHql = "USE " + config.getHiveDatabaseForIntermediateTable() + ";";
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(useDatabaseHql);
+ for (String delHive : allHiveTablesNeedToBeDeleted) {
+ hiveCmdBuilder.addStatement("drop table if exists " + delHive + "; ");
+ logger.info("Deleting Hive table " + delHive);
+ }
+ getCliCommandExecutor().execute(hiveCmdBuilder.build());
+
+ // If kylin.source.hive.keep-flat-table, some intermediate table might be kept.
+ // Do delete external path.
+ for (String tableToDelete : allHiveTablesNeedToBeDeleted) {
+ String uuid = tableToDelete.substring(tableToDelete.length() - uuidLength, tableToDelete.length());
+ String segmentId = uuid.replace("_", "-");
+
+ if (segmentId2JobId.containsKey(segmentId)) {
+ String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(),
+ segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
+ Path externalDataPath = new Path(path);
+ if (defaultFs.exists(externalDataPath)) {
+ defaultFs.delete(externalDataPath, true);
+ logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);
+ } else {
+ logger.info(
+ "Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)",
+ tableToDelete, path);
+ }
+ } else {
+ logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete,
+ segmentId2JobId.toString());
}
- System.out.println("----------------------------------------------------");
}
}
diff --git a/tool/src/test/java/org/apache/kylin/tool/MetadataCleanupJobTest.java b/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
similarity index 98%
rename from tool/src/test/java/org/apache/kylin/tool/MetadataCleanupJobTest.java
rename to server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
index ae3bef4..fa85a1e 100644
--- a/tool/src/test/java/org/apache/kylin/tool/MetadataCleanupJobTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/job/MetadataCleanupJobTest.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.kylin.tool;
+package org.apache.kylin.rest.job;
import static org.apache.kylin.common.util.LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA;
import static org.apache.kylin.common.util.LocalFileMetadataTestCase.cleanAfterClass;
diff --git a/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
index 4bb426b..e494884 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
@@ -35,6 +35,7 @@ import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.LocalFileMetadataTestCase.OverlayMetaHook;
import org.junit.After;
@@ -61,7 +62,7 @@ public class StorageCleanupJobTest {
prepareUnusedIntermediateHiveTable(mockFs);
prepareUnusedHDFSFiles(mockFs);
- MockStorageCleanupJob job = new MockStorageCleanupJob(mockFs, mockFs);
+ MockStorageCleanupJob job = new MockStorageCleanupJob(KylinConfig.getInstanceFromEnv(), mockFs, mockFs);
job.execute(new String[] { "--delete", "true" });
ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
@@ -104,8 +105,8 @@ public class StorageCleanupJobTest {
class MockStorageCleanupJob extends StorageCleanupJob {
- MockStorageCleanupJob(FileSystem defaultFs, FileSystem hbaseFs) {
- super(defaultFs, hbaseFs);
+ MockStorageCleanupJob(KylinConfig config, FileSystem defaultFs, FileSystem hbaseFs) {
+ super(config, defaultFs, hbaseFs);
}
@Override
diff --git a/tool/src/test/resources/test_meta/UUID b/server-base/src/test/resources/test_meta/UUID
similarity index 100%
rename from tool/src/test/resources/test_meta/UUID
rename to server-base/src/test/resources/test_meta/UUID
diff --git a/tool/src/test/resources/test_meta/cube/ci_inner_join_cube.json b/server-base/src/test/resources/test_meta/cube/ci_inner_join_cube.json
similarity index 100%
rename from tool/src/test/resources/test_meta/cube/ci_inner_join_cube.json
rename to server-base/src/test/resources/test_meta/cube/ci_inner_join_cube.json
diff --git a/tool/src/test/resources/test_meta/cube_desc/ci_inner_join_cube.json b/server-base/src/test/resources/test_meta/cube_desc/ci_inner_join_cube.json
similarity index 100%
rename from tool/src/test/resources/test_meta/cube_desc/ci_inner_join_cube.json
rename to server-base/src/test/resources/test_meta/cube_desc/ci_inner_join_cube.json
diff --git a/tool/src/test/resources/test_meta/cube_statistics/kylin_sales_cube/04b6db34-b243-4780-855a-4c7ea4cee817.seq b/server-base/src/test/resources/test_meta/cube_statistics/kylin_sales_cube/04b6db34-b243-4780-855a-4c7ea4cee817.seq
similarity index 100%
rename from tool/src/test/resources/test_meta/cube_statistics/kylin_sales_cube/04b6db34-b243-4780-855a-4c7ea4cee817.seq
rename to server-base/src/test/resources/test_meta/cube_statistics/kylin_sales_cube/04b6db34-b243-4780-855a-4c7ea4cee817.seq
diff --git a/tool/src/test/resources/test_meta/dict/DEFAULT.KYLIN_COUNTRY/NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict b/server-base/src/test/resources/test_meta/dict/DEFAULT.KYLIN_COUNTRY/NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict
similarity index 100%
rename from tool/src/test/resources/test_meta/dict/DEFAULT.KYLIN_COUNTRY/NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict
rename to server-base/src/test/resources/test_meta/dict/DEFAULT.KYLIN_COUNTRY/NAME/b58ce89d-3df6-46d1-8cff-2a9cfc8b7510.dict
diff --git a/tool/src/test/resources/test_meta/dict/DEFAULT.TEST_COUNTRY/NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict b/server-base/src/test/resources/test_meta/dict/DEFAULT.TEST_COUNTRY/NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict
similarity index 100%
rename from tool/src/test/resources/test_meta/dict/DEFAULT.TEST_COUNTRY/NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict
rename to server-base/src/test/resources/test_meta/dict/DEFAULT.TEST_COUNTRY/NAME/64ca8fea-b859-4e63-aea3-bfb4c6ee0c9d.dict
diff --git a/tool/src/test/resources/test_meta/execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930 b/server-base/src/test/resources/test_meta/execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930
similarity index 100%
rename from tool/src/test/resources/test_meta/execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930
rename to server-base/src/test/resources/test_meta/execute/d861b8b7-c773-47ab-bb1e-c8782ae8d930
diff --git a/tool/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930 b/server-base/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930
similarity index 100%
rename from tool/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930
rename to server-base/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930
diff --git a/tool/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-00 b/server-base/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-00
similarity index 100%
rename from tool/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-00
rename to server-base/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-00
diff --git a/tool/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-01 b/server-base/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-01
similarity index 100%
rename from tool/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-01
rename to server-base/src/test/resources/test_meta/execute_output/d861b8b7-c773-47ab-bb1e-c8782ae8d930-01
diff --git a/tool/src/test/resources/test_meta/kylin.properties b/server-base/src/test/resources/test_meta/kylin.properties
similarity index 100%
rename from tool/src/test/resources/test_meta/kylin.properties
rename to server-base/src/test/resources/test_meta/kylin.properties
diff --git a/tool/src/test/resources/test_meta/model_desc/ci_inner_join_model.json b/server-base/src/test/resources/test_meta/model_desc/ci_inner_join_model.json
similarity index 100%
rename from tool/src/test/resources/test_meta/model_desc/ci_inner_join_model.json
rename to server-base/src/test/resources/test_meta/model_desc/ci_inner_join_model.json
diff --git a/tool/src/test/resources/test_meta/project/default.json b/server-base/src/test/resources/test_meta/project/default.json
similarity index 100%
rename from tool/src/test/resources/test_meta/project/default.json
rename to server-base/src/test/resources/test_meta/project/default.json
diff --git a/tool/src/test/resources/test_meta/table/DEFAULT.TEST_ACCOUNT.json b/server-base/src/test/resources/test_meta/table/DEFAULT.TEST_ACCOUNT.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/DEFAULT.TEST_ACCOUNT.json
rename to server-base/src/test/resources/test_meta/table/DEFAULT.TEST_ACCOUNT.json
diff --git a/tool/src/test/resources/test_meta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/server-base/src/test/resources/test_meta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
rename to server-base/src/test/resources/test_meta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
diff --git a/tool/src/test/resources/test_meta/table/DEFAULT.TEST_COUNTRY.json b/server-base/src/test/resources/test_meta/table/DEFAULT.TEST_COUNTRY.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/DEFAULT.TEST_COUNTRY.json
rename to server-base/src/test/resources/test_meta/table/DEFAULT.TEST_COUNTRY.json
diff --git a/tool/src/test/resources/test_meta/table/DEFAULT.TEST_KYLIN_FACT.json b/server-base/src/test/resources/test_meta/table/DEFAULT.TEST_KYLIN_FACT.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/DEFAULT.TEST_KYLIN_FACT.json
rename to server-base/src/test/resources/test_meta/table/DEFAULT.TEST_KYLIN_FACT.json
diff --git a/tool/src/test/resources/test_meta/table/DEFAULT.TEST_ORDER.json b/server-base/src/test/resources/test_meta/table/DEFAULT.TEST_ORDER.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/DEFAULT.TEST_ORDER.json
rename to server-base/src/test/resources/test_meta/table/DEFAULT.TEST_ORDER.json
diff --git a/tool/src/test/resources/test_meta/table/EDW.TEST_CAL_DT.json b/server-base/src/test/resources/test_meta/table/EDW.TEST_CAL_DT.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/EDW.TEST_CAL_DT.json
rename to server-base/src/test/resources/test_meta/table/EDW.TEST_CAL_DT.json
diff --git a/tool/src/test/resources/test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json b/server-base/src/test/resources/test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json
rename to server-base/src/test/resources/test_meta/table/EDW.TEST_SELLER_TYPE_DIM.json
diff --git a/tool/src/test/resources/test_meta/table/EDW.TEST_SITES.json b/server-base/src/test/resources/test_meta/table/EDW.TEST_SITES.json
similarity index 100%
rename from tool/src/test/resources/test_meta/table/EDW.TEST_SITES.json
rename to server-base/src/test/resources/test_meta/table/EDW.TEST_SITES.json
diff --git a/tool/src/test/resources/test_meta/table_snapshot/DEFAULT.KYLIN_COUNTRY/92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot b/server-base/src/test/resources/test_meta/table_snapshot/DEFAULT.KYLIN_COUNTRY/92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot
similarity index 100%
rename from tool/src/test/resources/test_meta/table_snapshot/DEFAULT.KYLIN_COUNTRY/92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot
rename to server-base/src/test/resources/test_meta/table_snapshot/DEFAULT.KYLIN_COUNTRY/92456efe-9b79-4385-a5a3-e7f37b677bf7.snapshot
diff --git a/tool/src/test/resources/test_meta/table_snapshot/DEFAULT.TEST_COUNTRY/7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot b/server-base/src/test/resources/test_meta/table_snapshot/DEFAULT.TEST_COUNTRY/7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot
similarity index 100%
rename from tool/src/test/resources/test_meta/table_snapshot/DEFAULT.TEST_COUNTRY/7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot
rename to server-base/src/test/resources/test_meta/table_snapshot/DEFAULT.TEST_COUNTRY/7ecdb07b-a8d0-49d8-892b-fe2dd75512ca.snapshot
diff --git a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
index d1c8423..cf17f82 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MetadataCleanupJob.java
@@ -18,30 +18,17 @@
package org.apache.kylin.tool;
-import java.util.Arrays;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.job.dao.ExecutableDao;
-import org.apache.kylin.job.dao.ExecutableOutputPO;
-import org.apache.kylin.job.dao.ExecutablePO;
-import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class MetadataCleanupJob extends AbstractHadoopJob {
+public class MetadataCleanupJob extends AbstractApplication {
+ private static final Logger logger = LoggerFactory.getLogger(MetadataCleanupJob.class);
+ private static final int DEFAULT_JOB_OUTDATED_THRESHOLD_DAYS = 30; // 30 days
@SuppressWarnings("static-access")
private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false)
@@ -52,129 +39,27 @@ public class MetadataCleanupJob extends AbstractHadoopJob {
.isRequired(false).withDescription("Specify how many days of job metadata keeping. Default 30 days")
.create("jobThreshold");
- protected static final Logger logger = LoggerFactory.getLogger(MetadataCleanupJob.class);
- private boolean delete = false;
- private KylinConfig config = null;
- private static final long TIME_THREADSHOLD = 1 * 3600 * 1000L; // 1 hour
- private static final int DEFAULT_DAY_THREADSHOLD_FOR_JOB = 30; // 30 days
+ public static void main(String[] args) throws Exception {
+ new MetadataCleanupJob().execute(args);
+ }
@Override
- public int run(String[] args) throws Exception {
+ protected Options getOptions() {
Options options = new Options();
-
- logger.info("jobs args: " + Arrays.toString(args));
options.addOption(OPTION_DELETE);
options.addOption(OPTION_THRESHOLD_FOR_JOB);
- parseOptions(options, args);
-
- logger.info("options: '" + getOptionsAsString() + "'");
- logger.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
- logger.info("jobThreshold option value: '" + getOptionValue(OPTION_THRESHOLD_FOR_JOB) + "'");
- delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
- int jobThresholdDay = optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB)
- ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB))
- : DEFAULT_DAY_THREADSHOLD_FOR_JOB;
- cleanup(delete, jobThresholdDay);
- return 0;
+ return options;
}
- private ResourceStore getStore() {
- return ResourceStore.getStore(config);
- }
-
- private boolean isOlderThanThreshold(long resourceTime) {
- long currentTime = System.currentTimeMillis();
- return currentTime - resourceTime > TIME_THREADSHOLD;
- }
-
- public List<String> cleanup(boolean delete, int jobThresholdDay) throws Exception {
- config = KylinConfig.getInstanceFromEnv();
- CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<String> toDeleteResource = Lists.newArrayList();
-
- // two level resources, snapshot tables and cube statistics
- for (String resourceRoot : new String[]{ResourceStore.SNAPSHOT_RESOURCE_ROOT,
- ResourceStore.CUBE_STATISTICS_ROOT}) {
- NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);
-
- if (snapshotTables != null) {
- for (String snapshotTable : snapshotTables) {
- NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
- if (snapshotNames != null)
- for (String snapshot : snapshotNames) {
- if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
- toDeleteResource.add(snapshot);
- }
- }
- }
- }
-
- // three level resources, only dictionaries
- NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
-
- if (dictTables != null) {
- for (String table : dictTables) {
- NavigableSet<String> tableColNames = getStore().listResources(table);
- if (tableColNames != null)
- for (String tableCol : tableColNames) {
- NavigableSet<String> dictionaries = getStore().listResources(tableCol);
- if (dictionaries != null)
- for (String dict : dictionaries)
- if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
- toDeleteResource.add(dict);
- }
- }
- }
-
- Set<String> activeResourceList = Sets.newHashSet();
- for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
- for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
- activeResourceList.addAll(segment.getSnapshotPaths());
- activeResourceList.addAll(segment.getDictionaryPaths());
- activeResourceList.add(segment.getStatisticsResourcePath());
- }
- }
-
- toDeleteResource.removeAll(activeResourceList);
-
- // delete old and completed jobs
- ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
- List<ExecutablePO> allExecutable = executableDao.getJobs();
- for (ExecutablePO executable : allExecutable) {
- long lastModified = executable.getLastModified();
- ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
- long jobThresholdTime = jobThresholdDay * 24 * 3600 * 1000L;
-
- if (System.currentTimeMillis() - lastModified > jobThresholdTime
- && (ExecutableState.SUCCEED.toString().equals(output.getStatus())
- || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
- toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
- toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());
-
- for (ExecutablePO task : executable.getTasks()) {
- toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
- }
- }
- }
-
- if (toDeleteResource.size() > 0) {
- logger.info(
- "The following resources have no reference or is too old, will be cleaned from metadata store: \n");
-
- for (String s : toDeleteResource) {
- logger.info(s);
- if (delete) {
- getStore().deleteResource(s);
- }
- }
- } else {
- logger.info("No resource to be cleaned up from metadata store;");
- }
- return toDeleteResource;
- }
-
- public static void main(String[] args) throws Exception {
- int exitCode = ToolRunner.run(new MetadataCleanupJob(), args);
- System.exit(exitCode);
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ logger.info("delete option value: '" + optionsHelper.getOptionValue(OPTION_DELETE) + "'");
+ logger.info("jobThreshold option value: '" + optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB) + "'");
+ boolean delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE));
+ int jobOutdatedDays = optionsHelper.hasOption(OPTION_THRESHOLD_FOR_JOB)
+ ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_THRESHOLD_FOR_JOB))
+ : DEFAULT_JOB_OUTDATED_THRESHOLD_DAYS;
+
+ new org.apache.kylin.rest.job.MetadataCleanupJob().cleanup(delete, jobOutdatedDays);
}
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 03/07: minor, fix exception swallow in CubeDesc.init()
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9d88e47214b036b34dac38b12d549823cafdbefc
Author: Li Yang <li...@apache.org>
AuthorDate: Thu Apr 12 21:41:45 2018 +0800
minor, fix exception swallow in CubeDesc.init()
---
core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java | 6 +++---
core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java | 3 +--
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 1db66e1..78c71b7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -98,7 +98,7 @@ public class CubeDescManager {
cubeDesc.init(config);
} catch (Exception e) {
logger.warn("Broken cube desc " + cubeDesc.resourceName(), e);
- cubeDesc.addError(e.getMessage());
+ cubeDesc.addError(e.toString());
}
return cubeDesc;
}
@@ -193,7 +193,7 @@ public class CubeDescManager {
cubeDesc.init(config);
} catch (Exception e) {
logger.warn("Broken cube desc " + cubeDesc, e);
- cubeDesc.addError(e.getMessage());
+ cubeDesc.addError(e.toString());
}
postProcessCubeDesc(cubeDesc);
@@ -239,7 +239,7 @@ public class CubeDescManager {
desc.init(config);
} catch (Exception e) {
logger.warn("Broken cube desc " + desc, e);
- desc.addError(e.getMessage());
+ desc.addError(e.toString());
return desc;
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 93c327d..77b808b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -652,8 +652,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
Method initMeasureReferenceToColumnFamilyMethod = hbaseMappingAdapterClass.getMethod("initMeasureReferenceToColumnFamilyWithChecking", CubeDesc.class);
initMeasureReferenceToColumnFamilyMethod.invoke(null, this);
} catch (Exception e) {
- logger.error("Wrong configuration for kylin.metadata.hbasemapping-adapter: class "
- + hbaseMappingAdapterName + " not found. ");
+ throw new RuntimeException("Error during adapting hbase mapping", e);
}
} else {
if (hbaseMapping != null) {
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.