You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/26 07:49:37 UTC
[01/14] kylin git commit: minor, move QueryTransformer to Query module
Repository: kylin
Updated Branches:
refs/heads/master-hadoop3.0 [created] dcb7cf47c
minor, move QueryTransformer to Query module
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/037dd474
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/037dd474
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/037dd474
Branch: refs/heads/master-hadoop3.0
Commit: 037dd474980b2e33ee80674d4a5de2434a033e85
Parents: e9ec116
Author: lidongsjtu <li...@apache.org>
Authored: Tue Apr 11 18:47:02 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Tue Apr 11 19:00:34 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ClassUtil.java | 2 +
.../org/apache/kylin/cube/model/CubeDesc.java | 15 +-
.../test_case_data/localmeta/kylin.properties | 2 +-
.../query/util/CognosParenthesesEscape.java | 82 ++++++++
.../query/util/KeywordDefaultDirtyHack.java | 33 ++++
.../org/apache/kylin/query/util/QueryUtil.java | 190 +++++++++++++++++++
.../query/util/CognosParentesesEscapeTest.java | 61 ++++++
.../apache/kylin/query/util/QueryUtilTest.java | 61 ++++++
.../src/test/resources/query/cognos/query01.sql | 28 +++
.../resources/query/cognos/query01.sql.expected | 28 +++
.../apache/kylin/rest/service/QueryService.java | 4 +-
.../rest/util/CognosParenthesesEscape.java | 83 --------
.../rest/util/KeywordDefaultDirtyHack.java | 35 ----
.../org/apache/kylin/rest/util/QueryUtil.java | 190 -------------------
.../rest/util/CognosParentesesEscapeTest.java | 61 ------
.../apache/kylin/rest/util/QueryUtilTest.java | 65 -------
.../src/test/resources/query/cognos/query01.sql | 28 ---
.../resources/query/cognos/query01.sql.expected | 28 ---
.../rest/controller/QueryControllerTest.java | 2 +-
19 files changed, 502 insertions(+), 496 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 35a3277..0eb1af5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -65,6 +65,8 @@ public class ClassUtil {
classRenameMap.put("org.apache.kylin.job.cube.MergeDictionaryStep", "org.apache.kylin.engine.mr.steps.MergeDictionaryStep");
classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep");
classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterMergeStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterMergeStep");
+ classRenameMap.put("org.apache.kylin.rest.util.KeywordDefaultDirtyHack", "org.apache.kylin.query.util.KeywordDefaultDirtyHack");
+ classRenameMap.put("org.apache.kylin.rest.util.CognosParenthesesEscape", "org.apache.kylin.query.util.CognosParenthesesEscape");
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index b391055..820a172 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -97,7 +97,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
}
- public enum DeriveType implements java.io.Serializable{
+ public enum DeriveType implements java.io.Serializable {
LOOKUP, PK_FK, EXTENDED_COLUMN
}
@@ -485,7 +485,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion);
return true;
}
-
+
if (kylinVersion.isCompatibleWith(cubeVersion) && !kylinVersion.isSignatureCompatibleWith(cubeVersion)) {
logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion);
return true;
@@ -532,6 +532,17 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
}
+ public void deInit() {
+ config = null;
+ model = null;
+ allColumns = new LinkedHashSet<>();
+ allColumnDescs = new LinkedHashSet<>();
+ dimensionColumns = new LinkedHashSet<>();
+ derivedToHostMap = Maps.newHashMap();
+ hostToDerivedMap = Maps.newHashMap();
+ extendedColumnToHosts = Maps.newHashMap();
+ }
+
public void init(KylinConfig config) {
this.errors.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 9f7b24c..6fef7f7 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -80,7 +80,7 @@ kylin.engine.mr.uhc-reducer-count=3
### QUERY ###
-kylin.query.transformers=org.apache.kylin.rest.util.KeywordDefaultDirtyHack
+kylin.query.transformers=org.apache.kylin.query.util.KeywordDefaultDirtyHack
### SECURITY ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
new file mode 100644
index 0000000..70ab8da
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.util;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.common.util.Pair;
+
+/**
+ * from (a join b on a.x = b.y) join c
+ *
+ * similar in https://issues.apache.org/jira/browse/CALCITE-35
+ *
+ * we'll find such pattern and remove the parentheses
+ */
+public class CognosParenthesesEscape implements QueryUtil.IQueryTransformer {
+
+ private static final String S0 = "\\s*";
+ private static final String S1 = "\\s";
+ private static final String SM = "\\s+";
+ private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+";
+ private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?";
+ private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server
+ private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME;
+ private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // (
+ TABLE_NAME_WITH_OPTIONAL_ALIAS + // a
+ JOIN + SM + // join
+ TABLE_NAME_WITH_OPTIONAL_ALIAS + //b
+ SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2]
+ S0 + "\\)";// )
+ private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE);
+
+ private static int identifierNum = 0;
+
+ @Override
+ public String transform(String sql) {
+ Matcher m;
+ List<Pair<String, String>> matches = new LinkedList<>();
+ while (true) {
+ m = PARENTTHESES_PATTERN.matcher(sql);
+ if (!m.find())
+ break;
+
+ String oneParentheses = m.group(0);
+ String identifier = generateRandomName();
+ matches.add(new Pair<String, String>(identifier, oneParentheses.substring(1, oneParentheses.length() - 1)));
+ sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end());
+ }
+
+ for (int i = matches.size() - 1; i >= 0; i--) {
+ sql = sql.replaceAll(matches.get(i).getKey(), matches.get(i).getValue());
+ }
+
+ return sql;
+ }
+
+ private String generateRandomName() {
+ UUID uuid = UUID.randomUUID();
+ return uuid.toString().replace("-", "_") + "_" + (identifierNum++);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
new file mode 100644
index 0000000..e1398f6
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.util;
+
+public class KeywordDefaultDirtyHack implements QueryUtil.IQueryTransformer {
+
+ @Override
+ public String transform(String sql) {
+ // KYLIN-2108, DEFAULT is hive default database, but a sql keyword too, needs quote
+ sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
+ sql = sql.replace("default.", "\"default\".");
+ sql = sql.replace("defaultCatalog.", "");
+
+ return sql;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
new file mode 100644
index 0000000..d48a26f
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.util;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ */
+public class QueryUtil {
+
+ protected static final Logger logger = LoggerFactory.getLogger(QueryUtil.class);
+
+ private static List<IQueryTransformer> queryTransformers;
+
+ public interface IQueryTransformer {
+ String transform(String sql);
+ }
+
+ public static String massageSql(String sql) {
+ return massageSql(sql, 0, 0);
+ }
+
+ public static String massageSql(String sql, int limit, int offset) {
+ sql = sql.trim();
+ sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));
+
+ while (sql.endsWith(";"))
+ sql = sql.substring(0, sql.length() - 1);
+
+ if (limit > 0 && !sql.toLowerCase().contains("limit")) {
+ sql += ("\nLIMIT " + limit);
+ }
+
+ if (offset > 0 && !sql.toLowerCase().contains("offset")) {
+ sql += ("\nOFFSET " + offset);
+ }
+
+ // customizable SQL transformation
+ if (queryTransformers == null) {
+ initQueryTransformers();
+ }
+ for (IQueryTransformer t : queryTransformers) {
+ sql = t.transform(sql);
+ }
+ return sql;
+ }
+
+ private static void initQueryTransformers() {
+ List<IQueryTransformer> transformers = Lists.newArrayList();
+ transformers.add(new DefaultQueryTransformer());
+
+ String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers();
+ for (String clz : classes) {
+ try {
+ IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
+ transformers.add(t);
+ } catch (Exception e) {
+ logger.error("Failed to init query transformer", e);
+ }
+ }
+ queryTransformers = transformers;
+ }
+
+ // correct sick / invalid SQL
+ private static class DefaultQueryTransformer implements IQueryTransformer {
+
+ private static final String S0 = "\\s*";
+ private static final String S1 = "\\s";
+ private static final String SM = "\\s+";
+ private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
+ private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
+ private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
+ private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!=" + S0, Pattern.CASE_INSENSITIVE);
+ private static final Pattern PTN_INTERVAL = Pattern.compile("interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
+
+ @Override
+ public String transform(String sql) {
+ Matcher m;
+
+ // Case fn{ EXTRACT(...) }
+ // Use non-greedy regrex matching to remove escape functions
+ while (true) {
+ m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql);
+ if (!m.find())
+ break;
+ sql = sql.substring(0, m.start()) + m.group(1) + sql.substring(m.end());
+ }
+
+ // Case: HAVING COUNT(1)>0 without Group By
+ // Tableau generates: SELECT SUM(1) AS "COL" FROM "VAC_SW" HAVING
+ // COUNT(1)>0
+ m = PTN_HAVING_COUNT_GREATER_THAN_ZERO.matcher(sql);
+ if (m.find() && PTN_GROUP_BY.matcher(sql).find() == false) {
+ sql = sql.substring(0, m.start()) + " " + sql.substring(m.end());
+ }
+
+ // Case: SUM(1)
+ // Replace it with COUNT(1)
+ while (true) {
+ m = PTN_SUM_1.matcher(sql);
+ if (!m.find())
+ break;
+ sql = sql.substring(0, m.start()) + " COUNT(1) " + sql.substring(m.end());
+ }
+
+ // Case: !=
+ // Replace it with <>
+ while (true) {
+ m = PTN_NOT_EQ.matcher(sql);
+ if (!m.find())
+ break;
+ sql = sql.substring(0, m.start()) + " <> " + sql.substring(m.end());
+ }
+
+ // ( date '2001-09-28' + interval floor(1) day ) generated by cognos
+ // calcite only recognizes date '2001-09-28' + interval '1' day
+ while (true) {
+ m = PTN_INTERVAL.matcher(sql);
+ if (!m.find())
+ break;
+
+ int value = (int) Math.floor(Double.valueOf(m.group(2)));
+ sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3));
+ }
+
+ return sql;
+ }
+
+ }
+
+ public static String makeErrorMsgUserFriendly(Throwable e) {
+ String msg = e.getMessage();
+
+ // pick ParseException error message if possible
+ Throwable cause = e;
+ while (cause != null) {
+ if (cause.getClass().getName().contains("ParseException")) {
+ msg = cause.getMessage();
+ break;
+ }
+ cause = cause.getCause();
+ }
+
+ return makeErrorMsgUserFriendly(msg);
+ }
+
+ public static String makeErrorMsgUserFriendly(String errorMsg) {
+ try {
+ // make one line
+ errorMsg = errorMsg.replaceAll("\\s", " ");
+
+ // move cause to be ahead of sql, calcite creates the message pattern below
+ Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)");
+ Matcher matcher = pattern.matcher(errorMsg);
+ if (matcher.find()) {
+ return matcher.group(2).trim() + "\n" + "while executing SQL: \"" + matcher.group(1).trim() + "\"";
+ } else
+ return errorMsg;
+ } catch (Exception e) {
+ return errorMsg;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java b/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java
new file mode 100644
index 0000000..1f62d71
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/util/CognosParentesesEscapeTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.query.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Collection;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CognosParentesesEscapeTest {
+
+ @Test
+ public void basicTest() {
+ CognosParenthesesEscape escape = new CognosParenthesesEscape();
+ String data = "((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
+ String expected = "a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
+ String transformed = escape.transform(data);
+ Assert.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void advancedTest() throws IOException {
+ CognosParenthesesEscape escape = new CognosParenthesesEscape();
+ String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"), Charset.defaultCharset());
+ String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"), Charset.defaultCharset());
+ String transformed = escape.transform(query);
+ //System.out.println(transformed);
+ Assert.assertEquals(expected, transformed);
+ }
+
+ @Test
+ public void proguardTest() throws IOException {
+ CognosParenthesesEscape escape = new CognosParenthesesEscape();
+ Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" }, true);
+ for (File f : files) {
+ System.out.println("checking " + f.getAbsolutePath());
+ String query = FileUtils.readFileToString(f, Charset.defaultCharset());
+ String transformed = escape.transform(query);
+ Assert.assertEquals(query, transformed);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
new file mode 100644
index 0000000..a1edd89
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.util;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class QueryUtilTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testMassageSql() {
+ {
+ String sql = "select ( date '2001-09-28' + interval floor(1.2) day) from test_kylin_fact";
+ String s = QueryUtil.massageSql(sql, 0, 0);
+ Assert.assertEquals("select ( date '2001-09-28' + interval '1' day) from test_kylin_fact", s);
+ }
+ {
+ String sql = "select ( date '2001-09-28' + interval floor(2) month) from test_kylin_fact group by ( date '2001-09-28' + interval floor(2) month)";
+ String s = QueryUtil.massageSql(sql, 0, 0);
+ Assert.assertEquals("select ( date '2001-09-28' + interval '2' month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' month)", s);
+ }
+ }
+
+ @Test
+ public void testKeywordDefaultDirtyHack() {
+ {
+ String sql = "select * from DEFAULT.TEST_KYLIN_FACT";
+ String s = QueryUtil.massageSql(sql, 0, 0);
+ Assert.assertEquals("select * from \"DEFAULT\".TEST_KYLIN_FACT", s);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/resources/query/cognos/query01.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query01.sql b/query/src/test/resources/query/cognos/query01.sql
new file mode 100644
index 0000000..a76b4e1
--- /dev/null
+++ b/query/src/test/resources/query/cognos/query01.sql
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT "TABLE1"."DIM1_1" "DIM1_1"
+ ,"TABLE2"."DIM2_1" "DIM2_1"
+ ,SUM("FACT"."M1") "M1"
+ ,SUM("FACT"."M2") "M2"
+ FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
+ "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1")
+ LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
+ ON "FACT"."FK_2" = "TABLE2"."PK_2"
+ GROUP BY "TABLE2"."DIM2_1"
+ ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/query/src/test/resources/query/cognos/query01.sql.expected
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/cognos/query01.sql.expected b/query/src/test/resources/query/cognos/query01.sql.expected
new file mode 100644
index 0000000..c8005fc
--- /dev/null
+++ b/query/src/test/resources/query/cognos/query01.sql.expected
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT "TABLE1"."DIM1_1" "DIM1_1"
+ ,"TABLE2"."DIM2_1" "DIM2_1"
+ ,SUM("FACT"."M1") "M1"
+ ,SUM("FACT"."M2") "M2"
+ FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
+ "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1"
+ LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
+ ON "FACT"."FK_2" = "TABLE2"."PK_2"
+ GROUP BY "TABLE2"."DIM2_1"
+ ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 6a38638..6fed9a8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -65,6 +65,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -76,7 +77,6 @@ import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
import org.apache.kylin.rest.util.TableauInterceptor;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -447,7 +447,7 @@ public class QueryService extends BasicService {
return fakeResponse;
}
- String correctedSql = QueryUtil.massageSql(sqlRequest);
+ String correctedSql = QueryUtil.massageSql(sqlRequest.getSql(), sqlRequest.getLimit(), sqlRequest.getOffset());
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java b/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
deleted file mode 100644
index 33f4b63..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/CognosParenthesesEscape.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.UUID;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer;
-
-/**
- * from (a join b on a.x = b.y) join c
- *
- * similar in https://issues.apache.org/jira/browse/CALCITE-35
- *
- * we'll find such pattern and remove the parentheses
- */
-public class CognosParenthesesEscape implements IQueryTransformer {
-
- private static final String S0 = "\\s*";
- private static final String S1 = "\\s";
- private static final String SM = "\\s+";
- private static final String TABLE_OR_COLUMN_NAME = "[\\w\\\"\\'\\.]+";
- private static final String TABLE_NAME_WITH_OPTIONAL_ALIAS = TABLE_OR_COLUMN_NAME + "((\\s+as)?\\s+" + TABLE_OR_COLUMN_NAME + ")?";
- private static final String JOIN = "(\\s+inner|\\s+((left|right|full)(\\s+outer)?))?\\s+join";// as per http://stackoverflow.com/questions/406294/left-join-vs-left-outer-join-in-sql-server
- private static final String EQUAL_CONDITION = SM + TABLE_OR_COLUMN_NAME + S0 + "=" + S0 + TABLE_OR_COLUMN_NAME;
- private static final String PARENTHESE_PATTERN_STR = "\\(" + S0 + // (
- TABLE_NAME_WITH_OPTIONAL_ALIAS + // a
- JOIN + SM + // join
- TABLE_NAME_WITH_OPTIONAL_ALIAS + //b
- SM + "on" + EQUAL_CONDITION + "(\\s+and" + EQUAL_CONDITION + ")*" + // on a.x = b.y [and a.x2 = b.y2]
- S0 + "\\)";// )
- private static final Pattern PARENTTHESES_PATTERN = Pattern.compile(PARENTHESE_PATTERN_STR, Pattern.CASE_INSENSITIVE);
-
- private static int identifierNum = 0;
-
- @Override
- public String transform(String sql) {
- Matcher m;
- List<Pair<String, String>> matches = new LinkedList<>();
- while (true) {
- m = PARENTTHESES_PATTERN.matcher(sql);
- if (!m.find())
- break;
-
- String oneParentheses = m.group(0);
- String identifier = generateRandomName();
- matches.add(new Pair<String, String>(identifier, oneParentheses.substring(1, oneParentheses.length() - 1)));
- sql = sql.substring(0, m.start()) + identifier + sql.substring(m.end());
- }
-
- for (int i = matches.size() - 1; i >= 0; i--) {
- sql = sql.replaceAll(matches.get(i).getKey(), matches.get(i).getValue());
- }
-
- return sql;
- }
-
- private String generateRandomName() {
- UUID uuid = UUID.randomUUID();
- return uuid.toString().replace("-", "_") + "_" + (identifierNum++);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java b/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
deleted file mode 100644
index d8c8845..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/KeywordDefaultDirtyHack.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import org.apache.kylin.rest.util.QueryUtil.IQueryTransformer;
-
-public class KeywordDefaultDirtyHack implements IQueryTransformer {
-
- @Override
- public String transform(String sql) {
- // KYLIN-2108, DEFAULT is hive default database, but a sql keyword too, needs quote
- sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
- sql = sql.replace("default.", "\"default\".");
- sql = sql.replace("defaultCatalog.", "");
-
- return sql;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
deleted file mode 100644
index 26d6015..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class QueryUtil {
-
- protected static final Logger logger = LoggerFactory.getLogger(QueryUtil.class);
-
- private static List<IQueryTransformer> queryTransformers;
-
- public interface IQueryTransformer {
- String transform(String sql);
- }
-
- public static String massageSql(SQLRequest sqlRequest) {
- String sql = sqlRequest.getSql();
- sql = sql.trim();
- sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));
-
- while (sql.endsWith(";"))
- sql = sql.substring(0, sql.length() - 1);
-
- int limit = sqlRequest.getLimit();
- if (limit > 0 && !sql.toLowerCase().contains("limit")) {
- sql += ("\nLIMIT " + limit);
- }
-
- int offset = sqlRequest.getOffset();
- if (offset > 0 && !sql.toLowerCase().contains("offset")) {
- sql += ("\nOFFSET " + offset);
- }
-
- // customizable SQL transformation
- if (queryTransformers == null) {
- initQueryTransformers();
- }
- for (IQueryTransformer t : queryTransformers) {
- sql = t.transform(sql);
- }
- return sql;
- }
-
- private static void initQueryTransformers() {
- List<IQueryTransformer> transformers = Lists.newArrayList();
- transformers.add(new DefaultQueryTransformer());
-
- String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers();
- for (String clz : classes) {
- try {
- IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
- transformers.add(t);
- } catch (Exception e) {
- logger.error("Failed to init query transformer", e);
- }
- }
- queryTransformers = transformers;
- }
-
- // correct sick / invalid SQL
- private static class DefaultQueryTransformer implements IQueryTransformer {
-
- private static final String S0 = "\\s*";
- private static final String S1 = "\\s";
- private static final String SM = "\\s+";
- private static final Pattern PTN_GROUP_BY = Pattern.compile(S1 + "GROUP" + SM + "BY" + S1, Pattern.CASE_INSENSITIVE);
- private static final Pattern PTN_HAVING_COUNT_GREATER_THAN_ZERO = Pattern.compile(S1 + "HAVING" + SM + "[(]?" + S0 + "COUNT" + S0 + "[(]" + S0 + "1" + S0 + "[)]" + S0 + ">" + S0 + "0" + S0 + "[)]?", Pattern.CASE_INSENSITIVE);
- private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
- private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ S0, Pattern.CASE_INSENSITIVE);
- private static final Pattern PTN_INTERVAL = Pattern.compile("interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
- private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
-
- @Override
- public String transform(String sql) {
- Matcher m;
-
- // Case fn{ EXTRACT(...) }
- // Use non-greedy regrex matching to remove escape functions
- while (true) {
- m = PTN_HAVING_ESCAPE_FUNCTION.matcher(sql);
- if (!m.find())
- break;
- sql = sql.substring(0, m.start()) + m.group(1) + sql.substring(m.end());
- }
-
- // Case: HAVING COUNT(1)>0 without Group By
- // Tableau generates: SELECT SUM(1) AS "COL" FROM "VAC_SW" HAVING
- // COUNT(1)>0
- m = PTN_HAVING_COUNT_GREATER_THAN_ZERO.matcher(sql);
- if (m.find() && PTN_GROUP_BY.matcher(sql).find() == false) {
- sql = sql.substring(0, m.start()) + " " + sql.substring(m.end());
- }
-
- // Case: SUM(1)
- // Replace it with COUNT(1)
- while (true) {
- m = PTN_SUM_1.matcher(sql);
- if (!m.find())
- break;
- sql = sql.substring(0, m.start()) + " COUNT(1) " + sql.substring(m.end());
- }
-
- // Case: !=
- // Replace it with <>
- while (true) {
- m = PTN_NOT_EQ.matcher(sql);
- if (!m.find())
- break;
- sql = sql.substring(0, m.start()) + " <> " + sql.substring(m.end());
- }
-
- // ( date '2001-09-28' + interval floor(1) day ) generated by cognos
- // calcite only recognizes date '2001-09-28' + interval '1' day
- while (true) {
- m = PTN_INTERVAL.matcher(sql);
- if (!m.find())
- break;
-
- int value = (int) Math.floor(Double.valueOf(m.group(2)));
- sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3));
- }
-
- return sql;
- }
-
- }
-
- public static String makeErrorMsgUserFriendly(Throwable e) {
- String msg = e.getMessage();
-
- // pick ParseException error message if possible
- Throwable cause = e;
- while (cause != null) {
- if (cause.getClass().getName().contains("ParseException")) {
- msg = cause.getMessage();
- break;
- }
- cause = cause.getCause();
- }
-
- return makeErrorMsgUserFriendly(msg);
- }
-
- public static String makeErrorMsgUserFriendly(String errorMsg) {
- try {
- // make one line
- errorMsg = errorMsg.replaceAll("\\s", " ");
-
- // move cause to be ahead of sql, calcite creates the message pattern below
- Pattern pattern = Pattern.compile("error while executing SQL \"(.*)\":(.*)");
- Matcher matcher = pattern.matcher(errorMsg);
- if (matcher.find()) {
- return matcher.group(2).trim() + "\n" + "while executing SQL: \"" + matcher.group(1).trim() + "\"";
- } else
- return errorMsg;
- } catch (Exception e) {
- return errorMsg;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java
deleted file mode 100644
index 76df219..0000000
--- a/server-base/src/test/java/org/apache/kylin/rest/util/CognosParentesesEscapeTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.rest.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Collection;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class CognosParentesesEscapeTest {
-
- @Test
- public void basicTest() {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String data = "((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
- String expected = "a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
- String transformed = escape.transform(data);
- Assert.assertEquals(expected, transformed);
- }
-
- @Test
- public void advancedTest() throws IOException {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"), Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"), Charset.defaultCharset());
- String transformed = escape.transform(query);
- //System.out.println(transformed);
- Assert.assertEquals(expected, transformed);
- }
-
- @Test
- public void proguardTest() throws IOException {
- CognosParenthesesEscape escape = new CognosParenthesesEscape();
- Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" }, true);
- for (File f : files) {
- System.out.println("checking " + f.getAbsolutePath());
- String query = FileUtils.readFileToString(f, Charset.defaultCharset());
- String transformed = escape.transform(query);
- Assert.assertEquals(query, transformed);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
deleted file mode 100644
index dcd8c98..0000000
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class QueryUtilTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testMassageSql() {
- {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setSql("select ( date '2001-09-28' + interval floor(1.2) day) from test_kylin_fact");
- String s = QueryUtil.massageSql(sqlRequest);
- Assert.assertEquals("select ( date '2001-09-28' + interval '1' day) from test_kylin_fact", s);
- }
- {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setSql("select ( date '2001-09-28' + interval floor(2) month) from test_kylin_fact group by ( date '2001-09-28' + interval floor(2) month)");
- String s = QueryUtil.massageSql(sqlRequest);
- Assert.assertEquals("select ( date '2001-09-28' + interval '2' month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' month)", s);
- }
- }
-
- @Test
- public void testKeywordDefaultDirtyHack() {
- {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setSql("select * from DEFAULT.TEST_KYLIN_FACT");
- String s = QueryUtil.massageSql(sqlRequest);
- Assert.assertEquals("select * from \"DEFAULT\".TEST_KYLIN_FACT", s);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/resources/query/cognos/query01.sql
----------------------------------------------------------------------
diff --git a/server-base/src/test/resources/query/cognos/query01.sql b/server-base/src/test/resources/query/cognos/query01.sql
deleted file mode 100644
index a76b4e1..0000000
--- a/server-base/src/test/resources/query/cognos/query01.sql
+++ /dev/null
@@ -1,28 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "TABLE1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM ("COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
- "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1")
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server-base/src/test/resources/query/cognos/query01.sql.expected
----------------------------------------------------------------------
diff --git a/server-base/src/test/resources/query/cognos/query01.sql.expected b/server-base/src/test/resources/query/cognos/query01.sql.expected
deleted file mode 100644
index c8005fc..0000000
--- a/server-base/src/test/resources/query/cognos/query01.sql.expected
+++ /dev/null
@@ -1,28 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT "TABLE1"."DIM1_1" "DIM1_1"
- ,"TABLE2"."DIM2_1" "DIM2_1"
- ,SUM("FACT"."M1") "M1"
- ,SUM("FACT"."M2") "M2"
- FROM "COGNOS"."FACT" "FACT" LEFT OUTER JOIN "COGNOS"."TABLE1"
- "TABLE1" ON "FACT"."FK_1" = "TABLE1"."PK_1"
- LEFT OUTER JOIN "COGNOS"."TABLE2" "TABLE2"
- ON "FACT"."FK_2" = "TABLE2"."PK_2"
- GROUP BY "TABLE2"."DIM2_1"
- ,"TABLE1"."DIM1_1";
http://git-wip-us.apache.org/repos/asf/kylin/blob/037dd474/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
index e84235b..2043fef 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
@@ -19,12 +19,12 @@
package org.apache.kylin.rest.controller;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.request.MetaRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.service.ServiceTestBase;
-import org.apache.kylin.rest.util.QueryUtil;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
[08/14] kylin git commit: use tailored calcite-for-kylin
Posted by li...@apache.org.
use tailored calcite-for-kylin
more details at http://mail-archives.apache.org/mod_mbox/kylin-dev/201704.mbox/%3CCAF7etT=wEBPKm4C_6ffssQ0=kEhD=j1jz3O9DpjC+Zu9xWU=5A@mail.gmail.com%3E
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/410898f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/410898f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/410898f6
Branch: refs/heads/master-hadoop3.0
Commit: 410898f64632fe882fcd82072ed15f6d28e3e0aa
Parents: a1ad1eb
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Apr 18 21:29:35 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Apr 19 11:02:27 2017 +0800
----------------------------------------------------------------------
pom.xml | 65 +++++++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 49 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/410898f6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 40ccd0c..0e772ca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,7 +16,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -62,11 +63,11 @@
<!-- Spark versions -->
<spark.version>1.6.3</spark.version>
<kryo.version>4.0.0</kryo.version>
-
+
<!-- <reflections.version>0.9.10</reflections.version> -->
<!-- Calcite Version -->
- <calcite.version>1.12.0</calcite.version>
+ <calcite.version>1.12.0-kylin-r2-SNAPSHOT</calcite.version>
<avatica.version>1.9.0</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
@@ -115,7 +116,8 @@
<!-- REST Service -->
<spring.framework.version>3.2.17.RELEASE</spring.framework.version>
<spring.framework.security.version>3.1.2.RELEASE</spring.framework.security.version>
- <spring.framework.security.extensions.version>1.0.2.RELEASE</spring.framework.security.extensions.version>
+ <spring.framework.security.extensions.version>1.0.2.RELEASE
+ </spring.framework.security.extensions.version>
<opensaml.version>2.6.4</opensaml.version>
<aspectj.version>1.8.9</aspectj.version>
@@ -183,7 +185,8 @@
<scm>
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</connection>
- <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git</developerConnection>
+ <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/kylin.git
+ </developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/kylin.git</url>
<tag>HEAD</tag>
</scm>
@@ -847,6 +850,20 @@
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
+
+ <repository>
+ <id>nexus</id>
+ <name>Kyligence Repository</name>
+ <url>http://repository.kyligence.io:8081/repository/maven-public/
+ </url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ <updatePolicy>always</updatePolicy>
+ </snapshots>
+ </repository>
</repositories>
<build>
@@ -960,7 +977,8 @@
<phase>validate</phase>
<configuration>
<configLocation>dev-support/checkstyle.xml</configLocation>
- <suppressionsLocation>dev-support/checkstyle-suppressions.xml</suppressionsLocation>
+ <suppressionsLocation>dev-support/checkstyle-suppressions.xml
+ </suppressionsLocation>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<consoleOutput>true</consoleOutput>
<failsOnError>true</failsOnError>
@@ -1136,7 +1154,8 @@
<artifactItem>
<groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
- <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
+ <outputDirectory>${project.build.testOutputDirectory}
+ </outputDirectory>
<destFileName>jamm.jar</destFileName>
</artifactItem>
</artifactItems>
@@ -1179,7 +1198,8 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
<configuration>
- <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
+ <reportsDirectory>${project.basedir}/../target/surefire-reports
+ </reportsDirectory>
<excludes>
<exclude>**/IT*.java</exclude>
</excludes>
@@ -1190,10 +1210,14 @@
</property>
<property>
<name>log4j.configuration</name>
- <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
+ <value>
+ file:${project.basedir}/../build/conf/kylin-tools-log4j.properties
+ </value>
</property>
</systemProperties>
- <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
+ <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar
+ ${argLine} ${surefireArgLine}
+ </argLine>
</configuration>
</plugin>
</plugins>
@@ -1236,7 +1260,8 @@
<artifactItem>
<groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
- <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
+ <outputDirectory>${project.build.testOutputDirectory}
+ </outputDirectory>
<destFileName>jamm.jar</destFileName>
</artifactItem>
</artifactItems>
@@ -1279,7 +1304,8 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
<configuration>
- <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
+ <reportsDirectory>${project.basedir}/../target/surefire-reports
+ </reportsDirectory>
<excludes>
<exclude>**/IT*.java</exclude>
</excludes>
@@ -1290,10 +1316,14 @@
</property>
<property>
<name>log4j.configuration</name>
- <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
+ <value>
+ file:${project.basedir}/../build/conf/kylin-tools-log4j.properties
+ </value>
</property>
</systemProperties>
- <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
+ <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar
+ ${argLine} ${surefireArgLine}
+ </argLine>
</configuration>
</plugin>
</plugins>
@@ -1327,7 +1357,9 @@
<tarLongFileMode>posix</tarLongFileMode>
<runOnlyAtExecutionRoot>true</runOnlyAtExecutionRoot>
<appendAssemblyId>true</appendAssemblyId>
- <descriptor>assembly/src/main/config/assemblies/source-assembly.xml</descriptor>
+ <descriptor>
+ assembly/src/main/config/assemblies/source-assembly.xml
+ </descriptor>
<finalName>apache-kylin-${project.version}</finalName>
</configuration>
</execution>
@@ -1406,7 +1438,8 @@
<!-- MIT license -->
<exclude>webapp/app/css/AdminLTE.css</exclude>
<exclude>webapp/app/css/messenger-theme-ice.css</exclude>
- <exclude>webapp/app/js/directives/kylin_abn_tree_directive.js</exclude>
+ <exclude>webapp/app/js/directives/kylin_abn_tree_directive.js
+ </exclude>
<exclude>webapp/app/js/directives/angular-tree-control.js</exclude>
<exclude>webapp/app/js/directives/datetimepicker.js</exclude>
<exclude>webapp/app/js/directives/select.js</exclude>
[03/14] kylin git commit: KYLIN-2526 improve find-hive-dependency.sh
Posted by li...@apache.org.
KYLIN-2526 improve find-hive-dependency.sh
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d69cc278
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d69cc278
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d69cc278
Branch: refs/heads/master-hadoop3.0
Commit: d69cc2780d6849ff04d36e24882b932b22f6cc67
Parents: abb73ff
Author: Cheng Wang <ch...@kyligence.io>
Authored: Wed Apr 12 11:39:21 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Wed Apr 12 11:51:31 2017 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 51 +++++++++++++++++++++++++++++++++-
1 file changed, 50 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d69cc278/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index aa39da0..02a47ef 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -98,8 +98,57 @@ then
quit "hcatalog lib not found"
fi
+function checkFileExist()
+{
+ files=$1
+ misFiles=0
+ outputMissFiles=
+ for file in ${files//:/ }
+ do
+ let allFiles++
+ if [ ! -f "${file}" ]; then
+ outputMissFiles=${outputMissFiles}${file}", "
+ let misFiles++
+ fi
+ done
+ ratio=`echo "scale=3; ${misFiles}/${allFiles}" | bc`
+ [[ `echo "$ratio < 0.01" | bc ` -eq 1 ]] || quit "A couple of hive jars can't be found: ${outputMisFiles}!"
+}
+
+function validateDirectory()
+{
+ conf_path=$1
+ [[ -d "${conf_path}" ]] || quit "${conf_path} doesn't exist!"
+ unit=${conf_path: -1}
+ [[ "${unit}" == "/" ]] || conf_path=${conf_path}"/"
+
+ find="false"
+ filelist=`ls ${conf_path}`
+ for file in $filelist
+ do
+ if [ "${file}" == "hive-site.xml" ]
+ then
+ find="true"
+ break
+ fi
+ done
+ [[ "${find}" == "true" ]] || quit "ERROR, no hive-site.xml found under dir: ${conf_path}!"
+}
+
+if [ -z "$HIVE_LIB" ]
+then
+ verbose "HIVE_LIB is not set, try to retrieve hive lib from hive_exec_path"
+ hive_lib_dir="$(dirname $hive_exec_path)"
+else
+ hive_lib_dir="$HIVE_LIB"
+fi
+hive_lib=`find -L ${hive_lib_dir} -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+
+validateDirectory ${hive_conf_path}
+checkFileExist ${hive_lib}
+checkFileExist ${hcatalog}
-hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
verbose "hive dependency: $hive_dependency"
export hive_dependency
+
[07/14] kylin git commit: minor,
add some log in FactDistinctColumnsMapper
Posted by li...@apache.org.
minor, add some log in FactDistinctColumnsMapper
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1ad1eb2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1ad1eb2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1ad1eb2
Branch: refs/heads/master-hadoop3.0
Commit: a1ad1eb2d8f1e0fe4f4a5cdc2fd93f35ea82b2d4
Parents: bb1517b
Author: xiefan46 <95...@qq.com>
Authored: Tue Apr 18 15:30:59 2017 +0800
Committer: xiefan46 <95...@qq.com>
Committed: Tue Apr 18 15:30:59 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ad1eb2/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 993dafa..d36ae18 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -103,7 +103,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
-
TblColRef partitionColRef = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (partitionColRef != null) {
partitionColumnIndex = intermediateTableDesc.getColumnIndex(partitionColRef);
@@ -124,10 +123,12 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
row_hashcodes[i] = new ByteArray();
}
hf = Hashing.murmur3_32();
+ logger.info("Found KylinVersion : {}. Use old algorithm for cuboid sampling.", cubeDesc.getVersion());
} else {
isUsePutRowKeyToHllNewAlgorithm = true;
rowHashCodesLong = new long[nRowKey];
hf = Hashing.murmur3_128();
+ logger.info("Found KylinVersion : {}. Use new algorithm for cuboid sampling. About the details of the new algorithm, please refer to KYLIN-2518", cubeDesc.getVersion());
}
}
[10/14] kylin git commit: KYLIN-2555 Implicitly give
ADMIN=ADMIN+MODELER+ANALYST and MODELER=MODELER+ANALYST
Posted by li...@apache.org.
KYLIN-2555 Implicitly give ADMIN=ADMIN+MODELER+ANALYST and MODELER=MODELER+ANALYST
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3c70b8b9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3c70b8b9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3c70b8b9
Branch: refs/heads/master-hadoop3.0
Commit: 3c70b8b96176c58b784cda48afee8f560ace848f
Parents: 6d6e862
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Apr 19 19:19:18 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Apr 19 19:21:44 2017 +0800
----------------------------------------------------------------------
.../rest/security/AuthoritiesPopulator.java | 15 ++++++++----
.../apache/kylin/rest/service/AclService.java | 3 ++-
.../apache/kylin/rest/service/UserService.java | 5 ++++
server/src/main/resources/kylinSecurity.xml | 4 ++--
.../rest/controller/UserControllerTest.java | 3 ++-
.../kylin/rest/service/ServiceTestBase.java | 25 +++++++++++++++++++-
.../kylin/rest/service/UserServiceTest.java | 7 +++---
7 files changed, 49 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
index 7983fc0..2b290ce 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
@@ -21,6 +21,8 @@ package org.apache.kylin.rest.security;
import java.util.HashSet;
import java.util.Set;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.rest.constant.Constant;
import org.springframework.ldap.core.ContextSource;
import org.springframework.security.core.GrantedAuthority;
@@ -33,7 +35,6 @@ import org.springframework.security.ldap.userdetails.DefaultLdapAuthoritiesPopul
*/
public class AuthoritiesPopulator extends DefaultLdapAuthoritiesPopulator {
- String adminRole;
SimpleGrantedAuthority adminRoleAsAuthority;
SimpleGrantedAuthority adminAuthority = new SimpleGrantedAuthority(Constant.ROLE_ADMIN);
@@ -48,12 +49,12 @@ public class AuthoritiesPopulator extends DefaultLdapAuthoritiesPopulator {
*/
public AuthoritiesPopulator(ContextSource contextSource, String groupSearchBase, String adminRole, String defaultRole) {
super(contextSource, groupSearchBase);
- this.adminRole = adminRole;
this.adminRoleAsAuthority = new SimpleGrantedAuthority(adminRole);
- if (defaultRole.contains(Constant.ROLE_MODELER))
+ String[] defaultRoles = StringUtils.split(defaultRole, ",");
+ if (ArrayUtils.contains(defaultRoles, Constant.ROLE_MODELER))
this.defaultAuthorities.add(modelerAuthority);
- if (defaultRole.contains(Constant.ROLE_ANALYST))
+ if (ArrayUtils.contains(defaultRoles, Constant.ROLE_ANALYST))
this.defaultAuthorities.add(analystAuthority);
}
@@ -61,13 +62,17 @@ public class AuthoritiesPopulator extends DefaultLdapAuthoritiesPopulator {
public Set<GrantedAuthority> getGroupMembershipRoles(String userDn, String username) {
Set<GrantedAuthority> authorities = super.getGroupMembershipRoles(userDn, username);
+ authorities.addAll(defaultAuthorities);
+
if (authorities.contains(adminRoleAsAuthority)) {
authorities.add(adminAuthority);
authorities.add(modelerAuthority);
authorities.add(analystAuthority);
}
- authorities.addAll(defaultAuthorities);
+ if (authorities.contains(modelerAuthority)) {
+ authorities.add(analystAuthority);
+ }
return authorities;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index c0ece1d..b80d97d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -66,6 +66,7 @@ import org.springframework.security.acls.model.PermissionGrantingStrategy;
import org.springframework.security.acls.model.Sid;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.security.util.FieldUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
@@ -304,7 +305,7 @@ public class AclService implements MutableAclService {
String userName = psid.getPrincipal();
logger.debug("ACE SID name: " + userName);
if (!userService.userExists(userName))
- throw new NotFoundException("User : " + userName + " not exists. Please check or create user first");
+ throw new UsernameNotFoundException("User " + userName + " does not exist. Please make sure the user has logged in before");
}
AceInfo aceInfo = new AceInfo(ace);
put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index ab54882..9d94de1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -37,9 +37,11 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.security.AclHBaseStorage;
import org.apache.kylin.rest.util.Serializer;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
@@ -138,11 +140,13 @@ public class UserService implements UserDetailsManager {
}
@Override
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void createUser(UserDetails user) {
updateUser(user);
}
@Override
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void updateUser(UserDetails user) {
Table htable = null;
try {
@@ -162,6 +166,7 @@ public class UserService implements UserDetailsManager {
}
@Override
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void deleteUser(String username) {
Table htable = null;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index 3f4abdc..9d633ee 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -142,7 +142,7 @@
<scr:authentication-manager alias="testingAuthenticationManager">
<scr:authentication-provider>
<scr:user-service>
- <scr:user name="MODELER" password="$2a$10$Le5ernTeGNIARwMJsY0WaOLioNQdb0QD11DwjeyNqqNRp5NaDo2FG" authorities="ROLE_MODELER" />
+ <scr:user name="MODELER" password="$2a$10$Le5ernTeGNIARwMJsY0WaOLioNQdb0QD11DwjeyNqqNRp5NaDo2FG" authorities="ROLE_MODELER, ROLE_ANALYST" />
<scr:user name="ANALYST" password="$2a$10$s4INO3XHjPP5Vm2xH027Ce9QeXWdrfq5pvzuGr9z/lQmHqi0rsbNi" authorities="ROLE_ANALYST" />
<scr:user name="ADMIN" password="$2a$10$o3ktIWsGYxXNuUWQiYlZXOW5hWcqyNAFQsSSCSEWoC/BRVMAUjL32" authorities="ROLE_MODELER, ROLE_ANALYST, ROLE_ADMIN" />
</scr:user-service>
@@ -503,4 +503,4 @@
<bean id="parserPoolHolder" class="org.springframework.security.saml.parser.ParserPoolHolder"/>
</beans>
-</beans>
\ No newline at end of file
+</beans>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
index ab77a9a..767aaf1 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Assert;
import org.junit.Before;
@@ -46,7 +47,7 @@ public class UserControllerTest extends ServiceTestBase {
staticCreateTestMetadata();
List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
User user = new User("ADMIN", "ADMIN", authorities);
- Authentication authentication = new TestingAuthenticationToken(user, "ADMIN", "ROLE_ADMIN");
+ Authentication authentication = new TestingAuthenticationToken(user, "ADMIN", Constant.ROLE_ADMIN);
SecurityContextHolder.getContext().setAuthentication(authentication);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 3a587e4..a47fdd2 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -18,18 +18,23 @@
package org.apache.kylin.rest.service;
+import java.util.Arrays;
+
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.apache.kylin.rest.constant.Constant;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.User;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@@ -42,10 +47,13 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@ActiveProfiles("testing")
public class ServiceTestBase extends LocalFileMetadataTestCase {
+ @Autowired
+ UserService userService;
+
@BeforeClass
public static void setupResource() throws Exception {
staticCreateTestMetadata();
- Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", "ROLE_ADMIN");
+ Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
SecurityContextHolder.getContext().setAuthentication(authentication);
}
@@ -59,6 +67,21 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
Broadcaster.getInstance(config).notifyClearAll();
+
+ if (!userService.userExists("ADMIN")) {
+ userService.createUser(new User("ADMIN", "KYLIN", Arrays.asList(//
+ new UserService.UserGrantedAuthority(Constant.ROLE_ADMIN), new UserService.UserGrantedAuthority(Constant.ROLE_ANALYST), new UserService.UserGrantedAuthority(Constant.ROLE_MODELER))));
+ }
+
+ if (!userService.userExists("MODELER")) {
+ userService.createUser(new User("MODELER", "MODELER", Arrays.asList(//
+ new UserService.UserGrantedAuthority(Constant.ROLE_ANALYST), new UserService.UserGrantedAuthority(Constant.ROLE_MODELER))));
+ }
+
+ if (!userService.userExists("ROLE_ANALYST")) {
+ userService.createUser(new User("ROLE_ANALYST", "ROLE_ANALYST", Arrays.asList(//
+ new UserService.UserGrantedAuthority(Constant.ROLE_ANALYST))));
+ }
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/3c70b8b9/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
index 28515be..36c554e 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.service;
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.rest.constant.Constant;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -43,7 +44,7 @@ public class UserServiceTest extends ServiceTestBase {
Assert.assertTrue(!userService.userExists("ADMIN"));
List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
- authorities.add(new SimpleGrantedAuthority("ROLE_ADMIN"));
+ authorities.add(new SimpleGrantedAuthority(Constant.ROLE_ADMIN));
User user = new User("ADMIN", "PWD", authorities);
userService.createUser(user);
@@ -52,9 +53,9 @@ public class UserServiceTest extends ServiceTestBase {
UserDetails ud = userService.loadUserByUsername("ADMIN");
Assert.assertEquals("ADMIN", ud.getUsername());
Assert.assertEquals("PWD", ud.getPassword());
- Assert.assertEquals("ROLE_ADMIN", ud.getAuthorities().iterator().next().getAuthority());
+ Assert.assertEquals(Constant.ROLE_ADMIN, ud.getAuthorities().iterator().next().getAuthority());
Assert.assertEquals(1, ud.getAuthorities().size());
- Assert.assertTrue(userService.listUserAuthorities().contains("ROLE_ADMIN"));
+ Assert.assertTrue(userService.listUserAuthorities().contains(Constant.ROLE_ADMIN));
}
}
[02/14] kylin git commit: KYLIN-2542 use mvn package instead of mvn
install in package scripts
Posted by li...@apache.org.
KYLIN-2542 use mvn package instead of mvn install in package scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/abb73ff5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/abb73ff5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/abb73ff5
Branch: refs/heads/master-hadoop3.0
Commit: abb73ff560fcc0819ef7c1e252da00ef6f0256be
Parents: 037dd47
Author: lidongsjtu <li...@apache.org>
Authored: Tue Apr 11 23:19:18 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Apr 11 23:19:46 2017 +0800
----------------------------------------------------------------------
build/script/build.sh | 2 +-
build/script/package_diag.sh | 2 +-
build/script/package_libs.sh | 2 +-
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/abb73ff5/build/script/build.sh
----------------------------------------------------------------------
diff --git a/build/script/build.sh b/build/script/build.sh
index 458c612..6b1d767 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -22,7 +22,7 @@ echo "package back-end"
dir=$(dirname ${0})
cd ${dir}/../..
-mvn clean install -DskipTests $@ || { exit 1; }
+mvn clean package -DskipTests $@ || { exit 1; }
#package webapp
echo 'package front-end'
http://git-wip-us.apache.org/repos/asf/kylin/blob/abb73ff5/build/script/package_diag.sh
----------------------------------------------------------------------
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
index 5d163e3..5a41f63 100755
--- a/build/script/package_diag.sh
+++ b/build/script/package_diag.sh
@@ -49,7 +49,7 @@ fi
echo "kylin version: ${version}"
echo "package libraries"
-mvn clean install -DskipTests || { exit 1; }
+mvn clean package -DskipTests || { exit 1; }
echo "copy libraries"
rm -rf build/tool
http://git-wip-us.apache.org/repos/asf/kylin/blob/abb73ff5/build/script/package_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
index 7ef6b5d..01e52bc 100755
--- a/build/script/package_libs.sh
+++ b/build/script/package_libs.sh
@@ -70,7 +70,7 @@ EOF
git rev-parse HEAD >> build/commit_SHA1
echo "package libraries"
-mvn clean install -DskipTests || { exit 1; }
+mvn clean package -DskipTests || { exit 1; }
echo "copy libraries"
rm -rf build/lib
http://git-wip-us.apache.org/repos/asf/kylin/blob/abb73ff5/dev-support/test_all_against_hdp_2_4_0_0_169.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
index 2a3d24b..26ca825 100755
--- a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
+++ b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
@@ -21,5 +21,5 @@ dir=$(dirname ${0})
cd ${dir}
cd ..
-mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn clean package -DskipTests 2>&1 | tee mci.log
mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee mvnverify.log
[09/14] kylin git commit: KYLIN-2555 Check user exist before grant
authorities
Posted by li...@apache.org.
KYLIN-2555 Check user exist before grant authorities
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d6e862f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d6e862f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d6e862f
Branch: refs/heads/master-hadoop3.0
Commit: 6d6e862f15568a5cd40f3bf10f02645641e17d07
Parents: 410898f
Author: FAN XIE <xi...@outlook.com>
Authored: Wed Apr 19 12:34:16 2017 +0800
Committer: hongbin ma <ma...@kyligence.io>
Committed: Wed Apr 19 12:34:16 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/AclService.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6d6e862f/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 3e3efec..c0ece1d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -75,7 +75,6 @@ import com.fasterxml.jackson.databind.JsonMappingException;
/**
* @author xduo
- *
*/
@Component("aclService")
public class AclService implements MutableAclService {
@@ -111,6 +110,9 @@ public class AclService implements MutableAclService {
@Autowired
protected AclHBaseStorage aclHBaseStorage;
+ @Autowired
+ protected UserService userService;
+
public AclService() throws IOException {
fieldAces.setAccessible(true);
fieldAcl.setAccessible(true);
@@ -297,6 +299,13 @@ public class AclService implements MutableAclService {
}
for (AccessControlEntry ace : acl.getEntries()) {
+ if (ace.getSid() instanceof PrincipalSid) {
+ PrincipalSid psid = (PrincipalSid) ace.getSid();
+ String userName = psid.getPrincipal();
+ logger.debug("ACE SID name: " + userName);
+ if (!userService.userExists(userName))
+ throw new NotFoundException("User : " + userName + " not exists. Please check or create user first");
+ }
AceInfo aceInfo = new AceInfo(ace);
put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
@@ -315,6 +324,7 @@ public class AclService implements MutableAclService {
return (MutableAcl) readAclById(acl.getObjectIdentity());
}
+
private void genAces(List<Sid> sids, Result result, AclImpl acl) throws JsonParseException, JsonMappingException, IOException {
List<AceInfo> aceInfos = new ArrayList<AceInfo>();
if (null != sids) {
@@ -459,4 +469,5 @@ public class AclService implements MutableAclService {
}
}
+
}
[11/14] kylin git commit: KYLIN-2462 fix CubeDesc.getCopyOf()
Posted by li...@apache.org.
KYLIN-2462 fix CubeDesc.getCopyOf()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a8626030
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a8626030
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a8626030
Branch: refs/heads/master-hadoop3.0
Commit: a8626030668cb92abf7e4219041ca2df54779adf
Parents: 3c70b8b
Author: lidongsjtu <li...@apache.org>
Authored: Fri Apr 21 17:26:08 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Apr 21 17:26:08 2017 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a8626030/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0cbe9b7..53c11a3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1229,6 +1229,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
newCubeDesc.setOverrideKylinProps(cubeDesc.getOverrideKylinProps());
newCubeDesc.setConfig((KylinConfigExt) cubeDesc.getConfig());
newCubeDesc.setPartitionOffsetStart(cubeDesc.getPartitionOffsetStart());
+ newCubeDesc.setVersion(cubeDesc.getVersion());
newCubeDesc.updateRandomUuid();
return newCubeDesc;
}
[13/14] kylin git commit: KYLIN-2564 UsernameNotFoundException: User
XXX does not exist
Posted by li...@apache.org.
KYLIN-2564 UsernameNotFoundException: User XXX does not exist
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a0b537b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a0b537b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a0b537b8
Branch: refs/heads/master-hadoop3.0
Commit: a0b537b8ce5a2f5d8340e06f2a49b525fcef4572
Parents: 0762c4d
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Apr 25 18:19:51 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Apr 25 18:20:06 2017 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 1085 ++++++++++++----------
1 file changed, 598 insertions(+), 487 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0b537b8/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index 9d633ee..13d4f50 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -12,495 +12,606 @@
limitations under the License. See accompanying LICENSE file.
-->
-<beans xmlns="http://www.springframework.org/schema/beans" xmlns:tx="http://www.springframework.org/schema/tx"
- xmlns:scr="http://www.springframework.org/schema/security" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:context="http://www.springframework.org/schema/context" xsi:schemaLocation="http://www.springframework.org/schema/beans
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:scr="http://www.springframework.org/schema/security"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:util="http://www.springframework.org/schema/util" xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://www.springframework.org/schema/security
http://www.springframework.org/schema/security/spring-security-3.1.xsd
- http://www.springframework.org/schema/tx
- http://www.springframework.org/schema/tx/spring-tx-3.1.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
-
- <scr:global-method-security pre-post-annotations="enabled">
- <scr:expression-handler ref="expressionHandler" />
- </scr:global-method-security>
-
- <!-- acl config -->
- <bean id="aclPermissionFactory" class="org.apache.kylin.rest.security.AclPermissionFactory" />
-
- <bean id="expressionHandler" class="org.springframework.security.access.expression.method.DefaultMethodSecurityExpressionHandler">
- <property name="permissionEvaluator" ref="permissionEvaluator" />
- </bean>
-
- <bean id="permissionEvaluator" class="org.springframework.security.acls.AclPermissionEvaluator">
- <constructor-arg ref="aclService" />
- <property name="permissionFactory" ref="aclPermissionFactory" />
- </bean>
-
- <bean id="aclAuthorizationStrategy"
- class="org.springframework.security.acls.domain.AclAuthorizationStrategyImpl">
- <constructor-arg>
- <list>
- <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
- <constructor-arg value="ROLE_ADMIN" />
- </bean>
- <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
- <constructor-arg value="ROLE_ADMIN" />
- </bean>
- <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
- <constructor-arg value="ROLE_ADMIN" />
- </bean>
- </list>
- </constructor-arg>
- </bean>
-
- <bean id="auditLogger"
- class="org.springframework.security.acls.domain.ConsoleAuditLogger" />
-
- <bean id="permissionGrantingStrategy" class="org.springframework.security.acls.domain.DefaultPermissionGrantingStrategy">
- <constructor-arg ref="auditLogger" />
- </bean>
-
- <beans profile="ldap,saml">
- <bean id="ldapSource" class="org.springframework.security.ldap.DefaultSpringSecurityContextSource">
- <constructor-arg value="${kylin.security.ldap.connection-server}" />
- <property name="userDn" value="${kylin.security.ldap.connection-username}" />
- <property name="password" value="${kylin.security.ldap.connection-password}" />
- </bean>
-
- <bean id="kylinUserAuthProvider" class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
- <constructor-arg>
- <bean id="ldapUserAuthenticationProvider" class="org.springframework.security.ldap.authentication.LdapAuthenticationProvider">
- <constructor-arg>
- <bean class="org.springframework.security.ldap.authentication.BindAuthenticator">
- <constructor-arg ref="ldapSource" />
- <property name="userSearch">
- <bean id="userSearch" class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch">
- <constructor-arg index="0" value="${kylin.security.ldap.user-search-base}" />
- <constructor-arg index="1" value="${kylin.security.ldap.user-search-pattern}" />
- <constructor-arg index="2" ref="ldapSource" />
- </bean>
- </property>
- </bean>
- </constructor-arg>
- <constructor-arg>
- <bean class="org.apache.kylin.rest.security.AuthoritiesPopulator">
- <constructor-arg index="0" ref="ldapSource" />
- <constructor-arg index="1" value="${kylin.security.ldap.user-group-search-base}" />
- <constructor-arg index="2" value="${kylin.security.acl.admin-role}" />
- <constructor-arg index="3" value="${kylin.security.acl.default-role}" />
- </bean>
- </constructor-arg>
- </bean>
- </constructor-arg>
- </bean>
-
- <bean id="kylinServiceAccountAuthProvider" class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
- <constructor-arg>
- <bean id="ldapServiceAuthenticationProvider" class="org.springframework.security.ldap.authentication.LdapAuthenticationProvider">
- <constructor-arg>
- <bean class="org.springframework.security.ldap.authentication.BindAuthenticator">
- <constructor-arg ref="ldapSource" />
- <property name="userSearch">
- <bean id="userSearch" class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch">
- <constructor-arg index="0" value="${kylin.security.ldap.service-search-base}" />
- <constructor-arg index="1" value="${kylin.security.ldap.service-search-pattern}" />
- <constructor-arg index="2" ref="ldapSource" />
- </bean>
- </property>
- </bean>
- </constructor-arg>
- <constructor-arg>
- <bean class="org.apache.kylin.rest.security.AuthoritiesPopulator">
- <constructor-arg index="0" ref="ldapSource" />
- <constructor-arg index="1" value="${kylin.security.ldap.service-group-search-base}" />
- <constructor-arg index="2" value="${kylin.security.acl.admin-role}" />
- <constructor-arg index="3" value="${kylin.security.acl.default-role}" />
- </bean>
- </constructor-arg>
- </bean>
- </constructor-arg>
- </bean>
-
- </beans>
-
- <beans profile="ldap">
- <scr:authentication-manager alias="ldapAuthenticationManager">
- <!-- do user ldap auth -->
- <scr:authentication-provider ref="kylinUserAuthProvider"></scr:authentication-provider>
-
- <!-- do service account ldap auth -->
- <scr:authentication-provider ref="kylinServiceAccountAuthProvider"></scr:authentication-provider>
- </scr:authentication-manager>
-
- </beans>
-
- <beans profile="testing">
- <!-- user auth -->
- <bean id="passwordEncoder" class="org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder" />
-
- <scr:authentication-manager alias="testingAuthenticationManager">
- <scr:authentication-provider>
- <scr:user-service>
- <scr:user name="MODELER" password="$2a$10$Le5ernTeGNIARwMJsY0WaOLioNQdb0QD11DwjeyNqqNRp5NaDo2FG" authorities="ROLE_MODELER, ROLE_ANALYST" />
- <scr:user name="ANALYST" password="$2a$10$s4INO3XHjPP5Vm2xH027Ce9QeXWdrfq5pvzuGr9z/lQmHqi0rsbNi" authorities="ROLE_ANALYST" />
- <scr:user name="ADMIN" password="$2a$10$o3ktIWsGYxXNuUWQiYlZXOW5hWcqyNAFQsSSCSEWoC/BRVMAUjL32" authorities="ROLE_MODELER, ROLE_ANALYST, ROLE_ADMIN" />
- </scr:user-service>
- <scr:password-encoder ref="passwordEncoder" />
- </scr:authentication-provider>
- </scr:authentication-manager>
- </beans>
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.1.xsd
- <beans profile="testing,ldap">
- <scr:http auto-config="true" use-expressions="true">
- <scr:http-basic entry-point-ref="unauthorisedEntryPoint" />
-
- <scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll" />
- <scr:intercept-url pattern="/api/query*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/metadata*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/**/metrics" access="permitAll" />
- <scr:intercept-url pattern="/api/cache*/**" access="permitAll" />
- <scr:intercept-url pattern="/api/cubes/src/tables" access="hasAnyRole('ROLE_ANALYST')" />
- <scr:intercept-url pattern="/api/cubes*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/models*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/streaming*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/job*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/admin/config" access="permitAll" />
- <scr:intercept-url pattern="/api/projects" access="permitAll" />
- <scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')" />
- <scr:intercept-url pattern="/api/**" access="isAuthenticated()" />
-
- <scr:logout invalidate-session="true" delete-cookies="JSESSIONID" />
- <scr:session-management session-fixation-protection="newSession" />
- </scr:http>
- </beans>
-
- <beans profile="saml">
- <!-- Enable auto-wiring -->
- <context:annotation-config/>
-
- <!-- Scan for auto-wiring classes in spring saml packages -->
- <context:component-scan base-package="org.springframework.security.saml"/>
-
- <!-- Unsecured pages -->
- <scr:http security="none" pattern="/image/**"/>
- <scr:http security="none" pattern="/css/**"/>
- <scr:http security="none" pattern="/less/**"/>
- <scr:http security="none" pattern="/fonts/**"/>
- <scr:http security="none" pattern="/js/**"/>
- <scr:http security="none" pattern="/login/**"/>
- <scr:http security="none" pattern="/routes.json"/>
-
- <!-- Secured Rest API urls with LDAP basic authentication -->
- <scr:http pattern="/api/**" use-expressions="true" authentication-manager-ref="apiAccessAuthenticationManager">
- <scr:http-basic entry-point-ref="unauthorisedEntryPoint" />
-
- <scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll" />
- <scr:intercept-url pattern="/api/query*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/metadata*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/**/metrics" access="permitAll" />
- <scr:intercept-url pattern="/api/cache*/**" access="permitAll" />
- <scr:intercept-url pattern="/api/cubes/src/tables" access="hasAnyRole('ROLE_ANALYST')" />
- <scr:intercept-url pattern="/api/cubes*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/models*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/streaming*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/job*/**" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/admin/config" access="permitAll" />
- <scr:intercept-url pattern="/api/projects*/*" access="isAuthenticated()" />
- <scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')" />
- <scr:intercept-url pattern="/api/**" access="isAuthenticated()" />
-
- <scr:logout invalidate-session="true" delete-cookies="JSESSIONID" />
- <scr:session-management session-fixation-protection="newSession" />
- </scr:http>
-
- <!-- Secured non-api urls with SAML SSO -->
- <scr:http auto-config="true" entry-point-ref="samlEntryPoint" use-expressions="false" authentication-manager-ref="webAccessAuthenticationManager">
- <scr:intercept-url pattern="/**" access="IS_AUTHENTICATED_FULLY"/>
- <scr:custom-filter before="FIRST" ref="metadataGeneratorFilter"/>
- <scr:custom-filter after="BASIC_AUTH_FILTER" ref="samlFilter"/>
- </scr:http>
-
-
- <!-- API authentication manager -->
- <scr:authentication-manager id="apiAccessAuthenticationManager">
- <scr:authentication-provider ref="kylinServiceAccountAuthProvider" />
- <scr:authentication-provider ref="kylinUserAuthProvider" />
- </scr:authentication-manager>
-
-
- <!-- Web authentication manager -->
- <scr:authentication-manager id="webAccessAuthenticationManager">
- <scr:authentication-provider ref="kylinSAMLAuthenticationProvider"/>
- </scr:authentication-manager>
-
- <!-- Central storage of cryptographic keys -->
- <bean id="keyManager" class="org.springframework.security.saml.key.JKSKeyManager">
- <constructor-arg value="classpath:samlKeystore.jks"/>
- <constructor-arg type="java.lang.String" value="changeit"/>
- <constructor-arg>
- <map>
- <entry key="kylin" value="changeit"/>
- </map>
- </constructor-arg>
- <constructor-arg type="java.lang.String" value="kylin"/>
- </bean>
-
- <!-- Filters for processing of SAML messages -->
- <bean id="samlFilter" class="org.springframework.security.web.FilterChainProxy">
- <scr:filter-chain-map request-matcher="ant">
- <scr:filter-chain pattern="/saml/login/**" filters="samlEntryPoint"/>
- <scr:filter-chain pattern="/saml/logout/**" filters="samlLogoutFilter"/>
- <scr:filter-chain pattern="/saml/metadata/**" filters="metadataDisplayFilter"/>
- <scr:filter-chain pattern="/saml/SSO/**" filters="samlWebSSOProcessingFilter"/>
- <scr:filter-chain pattern="/saml/SSOHoK/**" filters="samlWebSSOHoKProcessingFilter"/>
- <scr:filter-chain pattern="/saml/SingleLogout/**" filters="samlLogoutProcessingFilter"/>
- </scr:filter-chain-map>
- </bean>
-
- <!-- Handler deciding where to redirect user after successful login -->
- <bean id="successRedirectHandler"
- class="org.springframework.security.web.authentication.SavedRequestAwareAuthenticationSuccessHandler">
- <property name="defaultTargetUrl" value="/models"/>
- </bean>
-
- <!-- Handler deciding where to redirect user after failed login -->
- <bean id="failureRedirectHandler"
- class="org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler">
- <property name="useForward" value="true"/>
- <property name="defaultFailureUrl" value="/login"/>
- </bean>
-
- <!-- Handler for successful logout -->
- <bean id="successLogoutHandler" class="org.springframework.security.web.authentication.logout.SimpleUrlLogoutSuccessHandler">
- <property name="defaultTargetUrl" value="/login"/>
- </bean>
-
- <!-- Logger for SAML messages and events -->
- <bean id="samlLogger" class="org.springframework.security.saml.log.SAMLDefaultLogger"/>
-
- <!-- Filter automatically generates default SP metadata -->
- <bean id="metadataGeneratorFilter" class="org.springframework.security.saml.metadata.MetadataGeneratorFilter">
- <constructor-arg>
- <bean class="org.springframework.security.saml.metadata.MetadataGenerator">
- <property name="extendedMetadata">
- <bean class="org.springframework.security.saml.metadata.ExtendedMetadata">
- <property name="idpDiscoveryEnabled" value="false"/>
- </bean>
- </property>
- <property name="entityBaseURL" value = "${kylin.security.saml.metadata-entity-base-url}"/>
- </bean>
- </constructor-arg>
- </bean>
-
- <!-- Entry point to initialize authentication, default values taken from properties file -->
- <bean id="samlEntryPoint" class="org.springframework.security.saml.SAMLEntryPoint">
- <property name="defaultProfileOptions">
- <bean class="org.springframework.security.saml.websso.WebSSOProfileOptions">
- <property name="includeScoping" value="false"/>
- </bean>
- </property>
- </bean>
-
- <!-- The filter is waiting for connections on URL suffixed with filterSuffix and presents SP metadata there -->
- <bean id="metadataDisplayFilter" class="org.springframework.security.saml.metadata.MetadataDisplayFilter"/>
-
- <!-- IDP Metadata configuration - paths to metadata of IDPs in circle of trust is here -->
- <bean id="metadata" class="org.springframework.security.saml.metadata.CachingMetadataManager">
- <constructor-arg>
- <list>
- <!-- Example of classpath metadata with Extended Metadata -->
- <bean class="org.springframework.security.saml.metadata.ExtendedMetadataDelegate">
- <constructor-arg>
- <bean class="org.opensaml.saml2.metadata.provider.FilesystemMetadataProvider">
- <constructor-arg>
- <value type="java.io.File">classpath:sso_metadata.xml</value>
- </constructor-arg>
- <property name="parserPool" ref="parserPool"/>
- </bean>
- </constructor-arg>
- <constructor-arg>
- <bean class="org.springframework.security.saml.metadata.ExtendedMetadata">
- </bean>
- </constructor-arg>
- <property name="metadataTrustCheck" value="false"/>
- </bean>
- </list>
- </constructor-arg>
- </bean>
-
- <bean id="ldapUserAuthoritiesPopulator" class="org.apache.kylin.rest.security.AuthoritiesPopulator">
- <constructor-arg index="0" ref="ldapSource" />
- <constructor-arg index="1" value="${kylin.security.ldap.user-group-search-base}" />
- <constructor-arg index="2" value="${kylin.security.acl.admin-role}" />
- <constructor-arg index="3" value="${kylin.security.acl.default-role}" />
- </bean>
-
- <bean id="userSearch" class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch">
- <constructor-arg index="0" value="${kylin.security.ldap.user-search-base}" />
- <constructor-arg index="1" value="${kylin.security.ldap.user-search-pattern}" />
- <constructor-arg index="2" ref="ldapSource" />
- </bean>
-
-
- <bean id="samlUserDetailsService" class="org.apache.kylin.rest.security.SAMLUserDetailsService">
- <constructor-arg>
- <bean id="ldapUserDetailsService" class="org.springframework.security.ldap.userdetails.LdapUserDetailsService">
- <constructor-arg ref="userSearch" />
- <constructor-arg ref="ldapUserAuthoritiesPopulator" />
- </bean>
- </constructor-arg>
- </bean>
-
- <bean id="kylinSAMLAuthenticationProvider" class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
- <constructor-arg>
- <!-- SAML Authentication Provider responsible for validating of received SAML messages -->
- <bean id="samlAuthenticationProvider" class="org.springframework.security.saml.SAMLAuthenticationProvider">
- <!-- OPTIONAL property: can be used to store/load user data after login -->
- <property name="userDetails" ref="samlUserDetailsService" />
- </bean>
- </constructor-arg>
- </bean>
-
-
- <!-- Provider of default SAML Context -->
- <!--
- <bean id="contextProvider" class="org.springframework.security.saml.context.SAMLContextProviderImpl"/>
- -->
-
- <!-- Provider of a SAML Context behind a LoadBanlancer or reverse proxy -->
- <bean id="contextProvider" class="org.springframework.security.saml.context.SAMLContextProviderLB">
- <property name="scheme" value="${kylin.security.saml.context-scheme}"/>
- <property name="serverName" value="${kylin.security.saml.context-server-name}"/>
- <property name="serverPort" value="${kylin.security.saml.context-server-port}"/>
- <property name="includeServerPortInRequestURL" value="false"/>
- <property name="contextPath" value="${kylin.security.saml.context-path}"/>
- </bean>
-
-
- <!-- Processing filter for WebSSO profile messages -->
- <bean id="samlWebSSOProcessingFilter" class="org.springframework.security.saml.SAMLProcessingFilter">
- <property name="authenticationManager" ref="webAccessAuthenticationManager"/>
- <property name="authenticationSuccessHandler" ref="successRedirectHandler"/>
- <property name="authenticationFailureHandler" ref="failureRedirectHandler"/>
- </bean>
-
- <!-- Processing filter for WebSSO Holder-of-Key profile -->
- <bean id="samlWebSSOHoKProcessingFilter" class="org.springframework.security.saml.SAMLWebSSOHoKProcessingFilter">
- <property name="authenticationManager" ref="webAccessAuthenticationManager"/>
- <property name="authenticationSuccessHandler" ref="successRedirectHandler"/>
- <property name="authenticationFailureHandler" ref="failureRedirectHandler"/>
- </bean>
-
- <!-- Logout handler terminating local session -->
- <bean id="logoutHandler"
- class="org.springframework.security.web.authentication.logout.SecurityContextLogoutHandler">
- <property name="invalidateHttpSession" value="false"/>
- </bean>
-
- <!-- Override default logout processing filter with the one processing SAML messages -->
- <bean id="samlLogoutFilter" class="org.springframework.security.saml.SAMLLogoutFilter">
- <constructor-arg index="0" ref="successLogoutHandler"/>
- <constructor-arg index="1" ref="logoutHandler"/>
- <constructor-arg index="2" ref="logoutHandler"/>
- </bean>
-
- <!-- Filter processing incoming logout messages -->
- <!-- First argument determines URL user will be redirected to after successful global logout -->
- <bean id="samlLogoutProcessingFilter" class="org.springframework.security.saml.SAMLLogoutProcessingFilter">
- <constructor-arg index="0" ref="successLogoutHandler"/>
- <constructor-arg index="1" ref="logoutHandler"/>
- </bean>
-
- <!-- Class loading incoming SAML messages from httpRequest stream -->
- <bean id="processor" class="org.springframework.security.saml.processor.SAMLProcessorImpl">
- <constructor-arg>
- <list>
- <ref bean="redirectBinding"/>
- <ref bean="postBinding"/>
- <ref bean="artifactBinding"/>
- <ref bean="soapBinding"/>
- <ref bean="paosBinding"/>
- </list>
- </constructor-arg>
- </bean>
-
- <!-- SAML 2.0 WebSSO Assertion Consumer -->
- <bean id="webSSOprofileConsumer" class="org.springframework.security.saml.websso.WebSSOProfileConsumerImpl">
- <property name="responseSkew" value="600"/> <!-- 10 minutes -->
- </bean>
-
- <!-- SAML 2.0 Holder-of-Key WebSSO Assertion Consumer -->
- <bean id="hokWebSSOprofileConsumer" class="org.springframework.security.saml.websso.WebSSOProfileConsumerHoKImpl"/>
-
- <!-- SAML 2.0 Web SSO profile -->
- <bean id="webSSOprofile" class="org.springframework.security.saml.websso.WebSSOProfileImpl"/>
-
- <!-- SAML 2.0 Holder-of-Key Web SSO profile -->
- <bean id="hokWebSSOProfile" class="org.springframework.security.saml.websso.WebSSOProfileConsumerHoKImpl"/>
-
- <!-- SAML 2.0 ECP profile -->
- <bean id="ecpprofile" class="org.springframework.security.saml.websso.WebSSOProfileECPImpl"/>
-
- <!-- SAML 2.0 Logout Profile -->
- <bean id="logoutprofile" class="org.springframework.security.saml.websso.SingleLogoutProfileImpl">
- <property name="responseSkew" value="600"/> <!-- 10 minutes -->
- </bean>
-
- <!-- Bindings, encoders and decoders used for creating and parsing messages -->
- <bean id="postBinding" class="org.springframework.security.saml.processor.HTTPPostBinding">
- <constructor-arg ref="parserPool"/>
- <constructor-arg ref="velocityEngine"/>
- </bean>
-
- <bean id="redirectBinding" class="org.springframework.security.saml.processor.HTTPRedirectDeflateBinding">
- <constructor-arg ref="parserPool"/>
- </bean>
-
- <bean id="artifactBinding" class="org.springframework.security.saml.processor.HTTPArtifactBinding">
- <constructor-arg ref="parserPool"/>
- <constructor-arg ref="velocityEngine"/>
- <constructor-arg>
- <bean class="org.springframework.security.saml.websso.ArtifactResolutionProfileImpl">
- <constructor-arg>
- <bean class="org.apache.commons.httpclient.HttpClient">
- <constructor-arg>
- <bean class="org.apache.commons.httpclient.MultiThreadedHttpConnectionManager"/>
- </constructor-arg>
- </bean>
- </constructor-arg>
- <property name="processor">
- <bean class="org.springframework.security.saml.processor.SAMLProcessorImpl">
- <constructor-arg ref="soapBinding"/>
- </bean>
- </property>
- </bean>
- </constructor-arg>
- </bean>
-
- <bean id="soapBinding" class="org.springframework.security.saml.processor.HTTPSOAP11Binding">
- <constructor-arg ref="parserPool"/>
- </bean>
-
- <bean id="paosBinding" class="org.springframework.security.saml.processor.HTTPPAOS11Binding">
- <constructor-arg ref="parserPool"/>
- </bean>
-
- <!-- Initialization of OpenSAML library-->
- <bean class="org.springframework.security.saml.SAMLBootstrap"/>
-
- <!-- Initialization of the velocity engine -->
- <bean id="velocityEngine" class="org.springframework.security.saml.util.VelocityFactory" factory-method="getEngine"/>
-
- <!-- XML parser pool needed for OpenSAML parsing -->
- <bean id="parserPool" class="org.opensaml.xml.parse.StaticBasicParserPool" init-method="initialize">
- <property name="builderFeatures">
- <map>
- <entry key="http://apache.org/xml/features/dom/defer-node-expansion" value="false"/>
- </map>
- </property>
- </bean>
-
- <bean id="parserPoolHolder" class="org.springframework.security.saml.parser.ParserPoolHolder"/>
- </beans>
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
+
+ <scr:global-method-security pre-post-annotations="enabled">
+ <scr:expression-handler ref="expressionHandler"/>
+ </scr:global-method-security>
+
+ <!-- acl config -->
+ <bean id="aclPermissionFactory" class="org.apache.kylin.rest.security.AclPermissionFactory"/>
+
+ <bean id="expressionHandler"
+ class="org.springframework.security.access.expression.method.DefaultMethodSecurityExpressionHandler">
+ <property name="permissionEvaluator" ref="permissionEvaluator"/>
+ </bean>
+
+ <bean id="permissionEvaluator" class="org.springframework.security.acls.AclPermissionEvaluator">
+ <constructor-arg ref="aclService"/>
+ <property name="permissionFactory" ref="aclPermissionFactory"/>
+ </bean>
+
+ <bean id="aclAuthorizationStrategy"
+ class="org.springframework.security.acls.domain.AclAuthorizationStrategyImpl">
+ <constructor-arg>
+ <list>
+ <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
+ <constructor-arg value="ROLE_ADMIN"/>
+ </bean>
+ <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
+ <constructor-arg value="ROLE_ADMIN"/>
+ </bean>
+ <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
+ <constructor-arg value="ROLE_ADMIN"/>
+ </bean>
+ </list>
+ </constructor-arg>
+ </bean>
+
+ <bean id="auditLogger"
+ class="org.springframework.security.acls.domain.ConsoleAuditLogger"/>
+
+ <bean id="permissionGrantingStrategy"
+ class="org.springframework.security.acls.domain.DefaultPermissionGrantingStrategy">
+ <constructor-arg ref="auditLogger"/>
+ </bean>
+
+ <beans profile="ldap,saml">
+ <bean id="ldapSource"
+ class="org.springframework.security.ldap.DefaultSpringSecurityContextSource">
+ <constructor-arg value="${kylin.security.ldap.connection-server}"/>
+ <property name="userDn" value="${kylin.security.ldap.connection-username}"/>
+ <property name="password" value="${kylin.security.ldap.connection-password}"/>
+ </bean>
+
+ <bean id="kylinUserAuthProvider"
+ class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
+ <constructor-arg>
+ <bean id="ldapUserAuthenticationProvider"
+ class="org.springframework.security.ldap.authentication.LdapAuthenticationProvider">
+ <constructor-arg>
+ <bean class="org.springframework.security.ldap.authentication.BindAuthenticator">
+ <constructor-arg ref="ldapSource"/>
+ <property name="userSearch">
+ <bean id="userSearch"
+ class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch">
+ <constructor-arg index="0"
+ value="${kylin.security.ldap.user-search-base}"/>
+ <constructor-arg index="1"
+ value="${kylin.security.ldap.user-search-pattern}"/>
+ <constructor-arg index="2" ref="ldapSource"/>
+ </bean>
+ </property>
+ </bean>
+ </constructor-arg>
+ <constructor-arg>
+ <bean class="org.apache.kylin.rest.security.AuthoritiesPopulator">
+ <constructor-arg index="0" ref="ldapSource"/>
+ <constructor-arg index="1"
+ value="${kylin.security.ldap.user-group-search-base}"/>
+ <constructor-arg index="2" value="${kylin.security.acl.admin-role}"/>
+ <constructor-arg index="3" value="${kylin.security.acl.default-role}"/>
+ </bean>
+ </constructor-arg>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+ <bean id="kylinServiceAccountAuthProvider"
+ class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
+ <constructor-arg>
+ <bean id="ldapServiceAuthenticationProvider"
+ class="org.springframework.security.ldap.authentication.LdapAuthenticationProvider">
+ <constructor-arg>
+ <bean class="org.springframework.security.ldap.authentication.BindAuthenticator">
+ <constructor-arg ref="ldapSource"/>
+ <property name="userSearch">
+ <bean id="userSearch"
+ class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch">
+ <constructor-arg index="0"
+ value="${kylin.security.ldap.service-search-base}"/>
+ <constructor-arg index="1"
+ value="${kylin.security.ldap.service-search-pattern}"/>
+ <constructor-arg index="2" ref="ldapSource"/>
+ </bean>
+ </property>
+ </bean>
+ </constructor-arg>
+ <constructor-arg>
+ <bean class="org.apache.kylin.rest.security.AuthoritiesPopulator">
+ <constructor-arg index="0" ref="ldapSource"/>
+ <constructor-arg index="1"
+ value="${kylin.security.ldap.service-group-search-base}"/>
+ <constructor-arg index="2" value="${kylin.security.acl.admin-role}"/>
+ <constructor-arg index="3" value="${kylin.security.acl.default-role}"/>
+ </bean>
+ </constructor-arg>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+ </beans>
+
+ <beans profile="ldap">
+ <scr:authentication-manager alias="ldapAuthenticationManager">
+ <!-- do user ldap auth -->
+ <scr:authentication-provider ref="kylinUserAuthProvider"></scr:authentication-provider>
+
+ <!-- do service account ldap auth -->
+ <scr:authentication-provider
+ ref="kylinServiceAccountAuthProvider"></scr:authentication-provider>
+ </scr:authentication-manager>
+
+ </beans>
+
+
+ <beans profile="testing">
+ <util:list id="adminAuthorities"
+ value-type="org.springframework.security.core.authority.SimpleGrantedAuthority">
+ <value>ROLE_ADMIN</value>
+ <value>ROLE_MODELER</value>
+ <value>ROLE_ANALYST</value>
+ </util:list>
+ <util:list id="modelerAuthorities"
+ value-type="org.springframework.security.core.authority.SimpleGrantedAuthority">
+ <value>ROLE_MODELER</value>
+ <value>ROLE_ANALYST</value>
+ </util:list>
+ <util:list id="analystAuthorities"
+ value-type="org.springframework.security.core.authority.SimpleGrantedAuthority">
+ <value>ROLE_ANALYST</value>
+ </util:list>
+
+ <bean id="kylinUserAuthProvider"
+ class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
+ <constructor-arg>
+ <bean class="org.springframework.security.authentication.dao.DaoAuthenticationProvider">
+ <property name="userDetailsService">
+ <bean class="org.springframework.security.core.userdetails.memory.InMemoryDaoImpl">
+ <property name="userMap">
+ <bean class="org.springframework.security.core.userdetails.memory.UserMap">
+ <property name="users">
+ <util:map key-type="java.lang.String"
+ value-type="org.springframework.security.core.userdetails.User">
+ <entry key="admin">
+ <bean class="org.springframework.security.core.userdetails.User">
+ <constructor-arg value="ADMIN"/>
+ <constructor-arg
+ value="$2a$10$o3ktIWsGYxXNuUWQiYlZXOW5hWcqyNAFQsSSCSEWoC/BRVMAUjL32"/>
+ <constructor-arg ref="adminAuthorities"/>
+ </bean>
+ </entry>
+ <entry key="modeler">
+ <bean class="org.springframework.security.core.userdetails.User">
+ <constructor-arg value="MODELER"/>
+ <constructor-arg
+ value="$2a$10$Le5ernTeGNIARwMJsY0WaOLioNQdb0QD11DwjeyNqqNRp5NaDo2FG"/>
+ <constructor-arg ref="modelerAuthorities"/>
+ </bean>
+ </entry>
+ <entry key="analyst">
+ <bean class="org.springframework.security.core.userdetails.User">
+ <constructor-arg value="ANALYST"/>
+ <constructor-arg
+ value="$2a$10$s4INO3XHjPP5Vm2xH027Ce9QeXWdrfq5pvzuGr9z/lQmHqi0rsbNi"/>
+ <constructor-arg ref="analystAuthorities"/>
+ </bean>
+ </entry>
+ </util:map>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ <property name="passwordEncoder" ref="passwordEncoder"></property>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+ <!-- user auth -->
+ <bean id="passwordEncoder"
+ class="org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder"/>
+
+ <scr:authentication-manager alias="testingAuthenticationManager">
+ <!-- do user ldap auth -->
+ <scr:authentication-provider ref="kylinUserAuthProvider"></scr:authentication-provider>
+ </scr:authentication-manager>
+ </beans>
+
+
+ <beans profile="testing,ldap">
+ <scr:http auto-config="true" use-expressions="true">
+ <scr:http-basic entry-point-ref="unauthorisedEntryPoint"/>
+
+ <scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll"/>
+ <scr:intercept-url pattern="/api/query*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/metadata*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/**/metrics" access="permitAll"/>
+ <scr:intercept-url pattern="/api/cache*/**" access="permitAll"/>
+ <scr:intercept-url pattern="/api/cubes/src/tables" access="hasAnyRole('ROLE_ANALYST')"/>
+ <scr:intercept-url pattern="/api/cubes*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/models*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/streaming*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/job*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/admin/config" access="permitAll"/>
+ <scr:intercept-url pattern="/api/projects" access="permitAll"/>
+ <scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')"/>
+ <scr:intercept-url pattern="/api/**" access="isAuthenticated()"/>
+
+ <scr:logout invalidate-session="true" delete-cookies="JSESSIONID"/>
+ <scr:session-management session-fixation-protection="newSession"/>
+ </scr:http>
+ </beans>
+
+ <beans profile="saml">
+ <!-- Enable auto-wiring -->
+ <context:annotation-config/>
+
+ <!-- Scan for auto-wiring classes in spring saml packages -->
+ <context:component-scan base-package="org.springframework.security.saml"/>
+
+ <!-- Unsecured pages -->
+ <scr:http security="none" pattern="/image/**"/>
+ <scr:http security="none" pattern="/css/**"/>
+ <scr:http security="none" pattern="/less/**"/>
+ <scr:http security="none" pattern="/fonts/**"/>
+ <scr:http security="none" pattern="/js/**"/>
+ <scr:http security="none" pattern="/login/**"/>
+ <scr:http security="none" pattern="/routes.json"/>
+
+ <!-- Secured Rest API urls with LDAP basic authentication -->
+ <scr:http pattern="/api/**" use-expressions="true"
+ authentication-manager-ref="apiAccessAuthenticationManager">
+ <scr:http-basic entry-point-ref="unauthorisedEntryPoint"/>
+
+ <scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll"/>
+ <scr:intercept-url pattern="/api/query*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/metadata*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/**/metrics" access="permitAll"/>
+ <scr:intercept-url pattern="/api/cache*/**" access="permitAll"/>
+ <scr:intercept-url pattern="/api/cubes/src/tables" access="hasAnyRole('ROLE_ANALYST')"/>
+ <scr:intercept-url pattern="/api/cubes*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/models*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/streaming*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/job*/**" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/admin/config" access="permitAll"/>
+ <scr:intercept-url pattern="/api/projects*/*" access="isAuthenticated()"/>
+ <scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')"/>
+ <scr:intercept-url pattern="/api/**" access="isAuthenticated()"/>
+
+ <scr:logout invalidate-session="true" delete-cookies="JSESSIONID"/>
+ <scr:session-management session-fixation-protection="newSession"/>
+ </scr:http>
+
+ <!-- Secured non-api urls with SAML SSO -->
+ <scr:http auto-config="true" entry-point-ref="samlEntryPoint" use-expressions="false"
+ authentication-manager-ref="webAccessAuthenticationManager">
+ <scr:intercept-url pattern="/**" access="IS_AUTHENTICATED_FULLY"/>
+ <scr:custom-filter before="FIRST" ref="metadataGeneratorFilter"/>
+ <scr:custom-filter after="BASIC_AUTH_FILTER" ref="samlFilter"/>
+ </scr:http>
+
+
+ <!-- API authentication manager -->
+ <scr:authentication-manager id="apiAccessAuthenticationManager">
+ <scr:authentication-provider ref="kylinServiceAccountAuthProvider"/>
+ <scr:authentication-provider ref="kylinUserAuthProvider"/>
+ </scr:authentication-manager>
+
+
+ <!-- Web authentication manager -->
+ <scr:authentication-manager id="webAccessAuthenticationManager">
+ <scr:authentication-provider ref="kylinSAMLAuthenticationProvider"/>
+ </scr:authentication-manager>
+
+ <!-- Central storage of cryptographic keys -->
+ <bean id="keyManager" class="org.springframework.security.saml.key.JKSKeyManager">
+ <constructor-arg value="classpath:samlKeystore.jks"/>
+ <constructor-arg type="java.lang.String" value="changeit"/>
+ <constructor-arg>
+ <map>
+ <entry key="kylin" value="changeit"/>
+ </map>
+ </constructor-arg>
+ <constructor-arg type="java.lang.String" value="kylin"/>
+ </bean>
+
+ <!-- Filters for processing of SAML messages -->
+ <bean id="samlFilter" class="org.springframework.security.web.FilterChainProxy">
+ <scr:filter-chain-map request-matcher="ant">
+ <scr:filter-chain pattern="/saml/login/**" filters="samlEntryPoint"/>
+ <scr:filter-chain pattern="/saml/logout/**" filters="samlLogoutFilter"/>
+ <scr:filter-chain pattern="/saml/metadata/**" filters="metadataDisplayFilter"/>
+ <scr:filter-chain pattern="/saml/SSO/**" filters="samlWebSSOProcessingFilter"/>
+ <scr:filter-chain pattern="/saml/SSOHoK/**"
+ filters="samlWebSSOHoKProcessingFilter"/>
+ <scr:filter-chain pattern="/saml/SingleLogout/**"
+ filters="samlLogoutProcessingFilter"/>
+ </scr:filter-chain-map>
+ </bean>
+
+ <!-- Handler deciding where to redirect user after successful login -->
+ <bean id="successRedirectHandler"
+ class="org.springframework.security.web.authentication.SavedRequestAwareAuthenticationSuccessHandler">
+ <property name="defaultTargetUrl" value="/models"/>
+ </bean>
+
+ <!-- Handler deciding where to redirect user after failed login -->
+ <bean id="failureRedirectHandler"
+ class="org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler">
+ <property name="useForward" value="true"/>
+ <property name="defaultFailureUrl" value="/login"/>
+ </bean>
+
+ <!-- Handler for successful logout -->
+ <bean id="successLogoutHandler"
+ class="org.springframework.security.web.authentication.logout.SimpleUrlLogoutSuccessHandler">
+ <property name="defaultTargetUrl" value="/login"/>
+ </bean>
+
+ <!-- Logger for SAML messages and events -->
+ <bean id="samlLogger" class="org.springframework.security.saml.log.SAMLDefaultLogger"/>
+
+ <!-- Filter automatically generates default SP metadata -->
+ <bean id="metadataGeneratorFilter"
+ class="org.springframework.security.saml.metadata.MetadataGeneratorFilter">
+ <constructor-arg>
+ <bean class="org.springframework.security.saml.metadata.MetadataGenerator">
+ <property name="extendedMetadata">
+ <bean class="org.springframework.security.saml.metadata.ExtendedMetadata">
+ <property name="idpDiscoveryEnabled" value="false"/>
+ </bean>
+ </property>
+ <property name="entityBaseURL"
+ value="${kylin.security.saml.metadata-entity-base-url}"/>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+ <!-- Entry point to initialize authentication, default values taken from properties file -->
+ <bean id="samlEntryPoint" class="org.springframework.security.saml.SAMLEntryPoint">
+ <property name="defaultProfileOptions">
+ <bean class="org.springframework.security.saml.websso.WebSSOProfileOptions">
+ <property name="includeScoping" value="false"/>
+ </bean>
+ </property>
+ </bean>
+
+ <!-- The filter is waiting for connections on URL suffixed with filterSuffix and presents SP metadata there -->
+ <bean id="metadataDisplayFilter"
+ class="org.springframework.security.saml.metadata.MetadataDisplayFilter"/>
+
+ <!-- IDP Metadata configuration - paths to metadata of IDPs in circle of trust is here -->
+ <bean id="metadata"
+ class="org.springframework.security.saml.metadata.CachingMetadataManager">
+ <constructor-arg>
+ <list>
+ <!-- Example of classpath metadata with Extended Metadata -->
+ <bean class="org.springframework.security.saml.metadata.ExtendedMetadataDelegate">
+ <constructor-arg>
+ <bean class="org.opensaml.saml2.metadata.provider.FilesystemMetadataProvider">
+ <constructor-arg>
+ <value type="java.io.File">classpath:sso_metadata.xml</value>
+ </constructor-arg>
+ <property name="parserPool" ref="parserPool"/>
+ </bean>
+ </constructor-arg>
+ <constructor-arg>
+ <bean class="org.springframework.security.saml.metadata.ExtendedMetadata">
+ </bean>
+ </constructor-arg>
+ <property name="metadataTrustCheck" value="false"/>
+ </bean>
+ </list>
+ </constructor-arg>
+ </bean>
+
+ <bean id="ldapUserAuthoritiesPopulator"
+ class="org.apache.kylin.rest.security.AuthoritiesPopulator">
+ <constructor-arg index="0" ref="ldapSource"/>
+ <constructor-arg index="1" value="${kylin.security.ldap.user-group-search-base}"/>
+ <constructor-arg index="2" value="${kylin.security.acl.admin-role}"/>
+ <constructor-arg index="3" value="${kylin.security.acl.default-role}"/>
+ </bean>
+
+ <bean id="userSearch"
+ class="org.springframework.security.ldap.search.FilterBasedLdapUserSearch">
+ <constructor-arg index="0" value="${kylin.security.ldap.user-search-base}"/>
+ <constructor-arg index="1" value="${kylin.security.ldap.user-search-pattern}"/>
+ <constructor-arg index="2" ref="ldapSource"/>
+ </bean>
+
+
+ <bean id="samlUserDetailsService"
+ class="org.apache.kylin.rest.security.SAMLUserDetailsService">
+ <constructor-arg>
+ <bean id="ldapUserDetailsService"
+ class="org.springframework.security.ldap.userdetails.LdapUserDetailsService">
+ <constructor-arg ref="userSearch"/>
+ <constructor-arg ref="ldapUserAuthoritiesPopulator"/>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+ <bean id="kylinSAMLAuthenticationProvider"
+ class="org.apache.kylin.rest.security.KylinAuthenticationProvider">
+ <constructor-arg>
+ <!-- SAML Authentication Provider responsible for validating of received SAML messages -->
+ <bean id="samlAuthenticationProvider"
+ class="org.springframework.security.saml.SAMLAuthenticationProvider">
+ <!-- OPTIONAL property: can be used to store/load user data after login -->
+ <property name="userDetails" ref="samlUserDetailsService"/>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+
+ <!-- Provider of default SAML Context -->
+ <!--
+ <bean id="contextProvider" class="org.springframework.security.saml.context.SAMLContextProviderImpl"/>
+ -->
+
+ <!-- Provider of a SAML Context behind a LoadBanlancer or reverse proxy -->
+ <bean id="contextProvider"
+ class="org.springframework.security.saml.context.SAMLContextProviderLB">
+ <property name="scheme" value="${kylin.security.saml.context-scheme}"/>
+ <property name="serverName" value="${kylin.security.saml.context-server-name}"/>
+ <property name="serverPort" value="${kylin.security.saml.context-server-port}"/>
+ <property name="includeServerPortInRequestURL" value="false"/>
+ <property name="contextPath" value="${kylin.security.saml.context-path}"/>
+ </bean>
+
+
+ <!-- Processing filter for WebSSO profile messages -->
+ <bean id="samlWebSSOProcessingFilter"
+ class="org.springframework.security.saml.SAMLProcessingFilter">
+ <property name="authenticationManager" ref="webAccessAuthenticationManager"/>
+ <property name="authenticationSuccessHandler" ref="successRedirectHandler"/>
+ <property name="authenticationFailureHandler" ref="failureRedirectHandler"/>
+ </bean>
+
+ <!-- Processing filter for WebSSO Holder-of-Key profile -->
+ <bean id="samlWebSSOHoKProcessingFilter"
+ class="org.springframework.security.saml.SAMLWebSSOHoKProcessingFilter">
+ <property name="authenticationManager" ref="webAccessAuthenticationManager"/>
+ <property name="authenticationSuccessHandler" ref="successRedirectHandler"/>
+ <property name="authenticationFailureHandler" ref="failureRedirectHandler"/>
+ </bean>
+
+ <!-- Logout handler terminating local session -->
+ <bean id="logoutHandler"
+ class="org.springframework.security.web.authentication.logout.SecurityContextLogoutHandler">
+ <property name="invalidateHttpSession" value="false"/>
+ </bean>
+
+ <!-- Override default logout processing filter with the one processing SAML messages -->
+ <bean id="samlLogoutFilter" class="org.springframework.security.saml.SAMLLogoutFilter">
+ <constructor-arg index="0" ref="successLogoutHandler"/>
+ <constructor-arg index="1" ref="logoutHandler"/>
+ <constructor-arg index="2" ref="logoutHandler"/>
+ </bean>
+
+ <!-- Filter processing incoming logout messages -->
+ <!-- First argument determines URL user will be redirected to after successful global logout -->
+ <bean id="samlLogoutProcessingFilter"
+ class="org.springframework.security.saml.SAMLLogoutProcessingFilter">
+ <constructor-arg index="0" ref="successLogoutHandler"/>
+ <constructor-arg index="1" ref="logoutHandler"/>
+ </bean>
+
+ <!-- Class loading incoming SAML messages from httpRequest stream -->
+ <bean id="processor" class="org.springframework.security.saml.processor.SAMLProcessorImpl">
+ <constructor-arg>
+ <list>
+ <ref bean="redirectBinding"/>
+ <ref bean="postBinding"/>
+ <ref bean="artifactBinding"/>
+ <ref bean="soapBinding"/>
+ <ref bean="paosBinding"/>
+ </list>
+ </constructor-arg>
+ </bean>
+
+ <!-- SAML 2.0 WebSSO Assertion Consumer -->
+ <bean id="webSSOprofileConsumer"
+ class="org.springframework.security.saml.websso.WebSSOProfileConsumerImpl">
+ <property name="responseSkew" value="600"/> <!-- 10 minutes -->
+ </bean>
+
+ <!-- SAML 2.0 Holder-of-Key WebSSO Assertion Consumer -->
+ <bean id="hokWebSSOprofileConsumer"
+ class="org.springframework.security.saml.websso.WebSSOProfileConsumerHoKImpl"/>
+
+ <!-- SAML 2.0 Web SSO profile -->
+ <bean id="webSSOprofile"
+ class="org.springframework.security.saml.websso.WebSSOProfileImpl"/>
+
+ <!-- SAML 2.0 Holder-of-Key Web SSO profile -->
+ <bean id="hokWebSSOProfile"
+ class="org.springframework.security.saml.websso.WebSSOProfileConsumerHoKImpl"/>
+
+ <!-- SAML 2.0 ECP profile -->
+ <bean id="ecpprofile"
+ class="org.springframework.security.saml.websso.WebSSOProfileECPImpl"/>
+
+ <!-- SAML 2.0 Logout Profile -->
+ <bean id="logoutprofile"
+ class="org.springframework.security.saml.websso.SingleLogoutProfileImpl">
+ <property name="responseSkew" value="600"/> <!-- 10 minutes -->
+ </bean>
+
+ <!-- Bindings, encoders and decoders used for creating and parsing messages -->
+ <bean id="postBinding" class="org.springframework.security.saml.processor.HTTPPostBinding">
+ <constructor-arg ref="parserPool"/>
+ <constructor-arg ref="velocityEngine"/>
+ </bean>
+
+ <bean id="redirectBinding"
+ class="org.springframework.security.saml.processor.HTTPRedirectDeflateBinding">
+ <constructor-arg ref="parserPool"/>
+ </bean>
+
+ <bean id="artifactBinding"
+ class="org.springframework.security.saml.processor.HTTPArtifactBinding">
+ <constructor-arg ref="parserPool"/>
+ <constructor-arg ref="velocityEngine"/>
+ <constructor-arg>
+ <bean class="org.springframework.security.saml.websso.ArtifactResolutionProfileImpl">
+ <constructor-arg>
+ <bean class="org.apache.commons.httpclient.HttpClient">
+ <constructor-arg>
+ <bean class="org.apache.commons.httpclient.MultiThreadedHttpConnectionManager"/>
+ </constructor-arg>
+ </bean>
+ </constructor-arg>
+ <property name="processor">
+ <bean class="org.springframework.security.saml.processor.SAMLProcessorImpl">
+ <constructor-arg ref="soapBinding"/>
+ </bean>
+ </property>
+ </bean>
+ </constructor-arg>
+ </bean>
+
+ <bean id="soapBinding"
+ class="org.springframework.security.saml.processor.HTTPSOAP11Binding">
+ <constructor-arg ref="parserPool"/>
+ </bean>
+
+ <bean id="paosBinding"
+ class="org.springframework.security.saml.processor.HTTPPAOS11Binding">
+ <constructor-arg ref="parserPool"/>
+ </bean>
+
+ <!-- Initialization of OpenSAML library-->
+ <bean class="org.springframework.security.saml.SAMLBootstrap"/>
+
+ <!-- Initialization of the velocity engine -->
+ <bean id="velocityEngine" class="org.springframework.security.saml.util.VelocityFactory"
+ factory-method="getEngine"/>
+
+ <!-- XML parser pool needed for OpenSAML parsing -->
+ <bean id="parserPool" class="org.opensaml.xml.parse.StaticBasicParserPool"
+ init-method="initialize">
+ <property name="builderFeatures">
+ <map>
+ <entry key="http://apache.org/xml/features/dom/defer-node-expansion"
+ value="false"/>
+ </map>
+ </property>
+ </bean>
+
+ <bean id="parserPoolHolder"
+ class="org.springframework.security.saml.parser.ParserPoolHolder"/>
+ </beans>
</beans>
[05/14] kylin git commit: KYLIN-2462 fix PK/FK backward-compatibility
issue (By LiYang)
Posted by li...@apache.org.
KYLIN-2462 fix PK/FK backward-compatibility issue (By LiYang)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/675c0d82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/675c0d82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/675c0d82
Branch: refs/heads/master-hadoop3.0
Commit: 675c0d82a0e568cf6b7a4d36ae12fc8966d5d705
Parents: 8733dda
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 13 20:00:26 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 13 20:04:42 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinVersion.java | 47 +++++++++++++++-----
.../org/apache/kylin/cube/model/CubeDesc.java | 13 ++++++
2 files changed, 50 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/675c0d82/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 465ca5a..14fcfd0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -33,7 +33,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
-public class KylinVersion {
+public class KylinVersion implements Comparable {
private static final String COMMIT_SHA1_v15 = "commit_SHA1";
private static final String COMMIT_SHA1_v13 = "commit.sha1";
@@ -66,22 +66,44 @@ public class KylinVersion {
return "" + major + "." + minor + "." + revision;
}
+ @Override
+ public int compareTo(Object o) {
+ KylinVersion v = (KylinVersion) o;
+ int comp;
+
+ comp = this.major - v.major;
+ if (comp != 0)
+ return comp;
+
+ comp = this.minor - v.minor;
+ if (comp != 0)
+ return comp;
+
+ comp = this.revision - v.revision;
+ if (comp != 0)
+ return comp;
+
+ return (this.isSnapshot ? 0 : 1) - (v.isSnapshot ? 0 : 1);
+ }
+
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.0.0");
+ private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
+
private static final Set<KylinVersion> SIGNATURE_INCOMPATIBLE_REVISIONS = new HashSet<KylinVersion>();
/**
- * 1.5.1 is actually compatible with 1.5.0's cube. However the "calculate signature" method in 1.5.1 code base somehow
- * gives different signature values for 1.5.0 cubes. To prevent from users having to take care of this mess, as people
- * usually won't expect to take lots of efforts for small upgrade (from 1.5.0 to 1.5.1), a special list of
+ * 1.5.1 is actually compatible with 1.5.0's cube. However the "calculate signature" method in 1.5.1 code base somehow
+ * gives different signature values for 1.5.0 cubes. To prevent from users having to take care of this mess, as people
+ * usually won't expect to take lots of efforts for small upgrade (from 1.5.0 to 1.5.1), a special list of
* SIGNATURE_INCOMPATIBLE_REVISIONS is introduced to silently take care of such legacy cubes.
- *
+ *
* We should NEVER add new stuff to SIGNATURE_INCOMPATIBLE_REVISIONS. "calculate signature" should always return consistent values
* to compatible versions. If it's impossible to maintain consistent signatures between upgrade, we should increase the minor version,
- * e.g. it's better to skip 1.5.1 and use 1.6.0 as the next release version to 1.5.0, or even to use 2.0.0, as people tends to accept
+ * e.g. it's better to skip 1.5.1 and use 1.6.0 as the next release version to 1.5.0, or even to use 2.0.0, as people tends to accept
* doing more (e.g. Having to use sth like a metastore upgrade tool when upgrading Kylin)
*/
static {
@@ -100,6 +122,14 @@ public class KylinVersion {
return CURRENT_KYLIN_VERSION;
}
+ public static boolean isBefore200(String ver) {
+ return new KylinVersion(ver).compareTo(VERSION_200) < 0;
+ }
+
+ public static void main(String[] args) {
+ System.out.println(getKylinClientInformation());
+ }
+
public boolean isCompatibleWith(KylinVersion v) {
KylinVersion current = CURRENT_KYLIN_VERSION;
if (current.major != v.major || current.minor != v.minor) {
@@ -135,10 +165,6 @@ public class KylinVersion {
return !signatureIncompatible;
}
- public static void main(String[] args) {
- System.out.println(getKylinClientInformation());
- }
-
public static String getKylinClientInformation() {
StringBuilder buf = new StringBuilder();
@@ -172,4 +198,5 @@ public class KylinVersion {
return StringUtils.EMPTY;
}
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/675c0d82/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 820a172..0cbe9b7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -875,6 +875,19 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
TblColRef col = model.findColumn(dim.getTable(), colName);
+
+ // for backward compatibility
+ if (KylinVersion.isBefore200(getVersion())) {
+ // always use FK instead PK, FK could be shared by more than one lookup tables
+ JoinDesc join = dim.getJoin();
+ if (join != null) {
+ int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), col);
+ if (idx >= 0) {
+ col = join.getForeignKeyColumns()[idx];
+ }
+ }
+ }
+
return initDimensionColRef(col);
}
[06/14] kylin git commit: KYLIN-2552 Fix backward compatibility (#617)
Posted by li...@apache.org.
KYLIN-2552 Fix backward compatibility (#617)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bb1517bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bb1517bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bb1517bb
Branch: refs/heads/master-hadoop3.0
Commit: bb1517bb56cd9230836ab2df807680e960aa943a
Parents: 675c0d8
Author: FAN XIE <xi...@outlook.com>
Authored: Tue Apr 18 15:20:00 2017 +0800
Committer: hongbin ma <ma...@kyligence.io>
Committed: Tue Apr 18 15:20:00 2017 +0800
----------------------------------------------------------------------
.../mr/steps/FactDistinctColumnsMapper.java | 54 ++++++++++++++++++--
1 file changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bb1517bb/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index e6cea2b..993dafa 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.List;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.KylinVersion;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
@@ -64,6 +66,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
private int samplingPercentage;
//private ByteArray[] row_hashcodes = null;
private long[] rowHashCodesLong = null;
+ private ByteArray[] row_hashcodes = null;
private ByteBuffer tmpbuf;
private static final Text EMPTY_TEXT = new Text();
public static final byte MARK_FOR_PARTITION_COL = (byte) 0xFE;
@@ -74,6 +77,9 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
private SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ //about details of the new algorithm, please see KYLIN-2518
+ private boolean isUsePutRowKeyToHllNewAlgorithm;
+
@Override
protected void setup(Context context) throws IOException {
super.setup(context);
@@ -96,8 +102,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision(), RegisterType.DENSE);
}
- hf = Hashing.murmur3_128();
- rowHashCodesLong = new long[nRowKey];
+
TblColRef partitionColRef = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (partitionColRef != null) {
@@ -111,7 +116,21 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
} else {
needFetchPartitionCol = true;
}
+ //for KYLIN-2518 backward compatibility
+ if (KylinVersion.isBefore200(cubeDesc.getVersion())) {
+ isUsePutRowKeyToHllNewAlgorithm = false;
+ row_hashcodes = new ByteArray[nRowKey];
+ for (int i = 0; i < nRowKey; i++) {
+ row_hashcodes[i] = new ByteArray();
+ }
+ hf = Hashing.murmur3_32();
+ } else {
+ isUsePutRowKeyToHllNewAlgorithm = true;
+ rowHashCodesLong = new long[nRowKey];
+ hf = Hashing.murmur3_128();
+ }
}
+
}
private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet, List<Long> allCuboids) {
@@ -176,7 +195,11 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
if (collectStatistics) {
if (rowCount % 100 < samplingPercentage) {
- putRowKeyToHLL(row);
+ if (isUsePutRowKeyToHllNewAlgorithm) {
+ putRowKeyToHLLNew(row);
+ } else {
+ putRowKeyToHLLOld(row);
+ }
}
if (needFetchPartitionCol == true) {
@@ -208,7 +231,30 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
return size;
}
- private void putRowKeyToHLL(String[] row) {
+ private void putRowKeyToHLLOld(String[] row) {
+ //generate hash for each row key column
+ for (int i = 0; i < nRowKey; i++) {
+ Hasher hc = hf.newHasher();
+ String colValue = row[intermediateTableDesc.getRowKeyColumnIndexes()[i]];
+ if (colValue != null) {
+ row_hashcodes[i].set(hc.putString(colValue).hash().asBytes());
+ } else {
+ row_hashcodes[i].set(hc.putInt(0).hash().asBytes());
+ }
+ }
+
+ // user the row key column hash to get a consolidated hash for each cuboid
+ for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
+ Hasher hc = hf.newHasher();
+ for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
+ hc.putBytes(row_hashcodes[allCuboidsBitSet[i][position]].array());
+ }
+
+ allCuboidsHLL[i].add(hc.hash().asBytes());
+ }
+ }
+
+ private void putRowKeyToHLLNew(String[] row) {
//generate hash for each row key column
for (int i = 0; i < nRowKey; i++) {
Hasher hc = hf.newHasher();
[04/14] kylin git commit: Make stopped jobs discardable. (#571)
Posted by li...@apache.org.
Make stopped jobs discardable. (#571)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8733ddab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8733ddab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8733ddab
Branch: refs/heads/master-hadoop3.0
Commit: 8733ddab7b710ce6807ba06a4fc000679453e379
Parents: d69cc27
Author: nichunen <ch...@kyligence.io>
Authored: Wed Apr 12 14:24:50 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Wed Apr 12 14:44:45 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/jobs/jobList.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8733ddab/webapp/app/partials/jobs/jobList.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/jobs/jobList.html b/webapp/app/partials/jobs/jobList.html
index d286022..4d65ac1 100644
--- a/webapp/app/partials/jobs/jobList.html
+++ b/webapp/app/partials/jobs/jobList.html
@@ -139,7 +139,7 @@
<li ng-if="job.job_status=='RUNNING' || job.job_status=='NEW' || job.job_status=='PENDING'">
<a ng-click="pause(job)">Pause</a>
</li>
- <li ng-if="job.job_status=='RUNNING' || job.job_status=='NEW' || job.job_status=='PENDING' || job.job_status=='ERROR'">
+ <li ng-if="job.job_status=='RUNNING' || job.job_status=='NEW' || job.job_status=='PENDING' || job.job_status=='ERROR' || job.job_status=='STOPPED'">
<a ng-click="cancel(job)">Discard</a>
</li>
<li>
[12/14] kylin git commit: minior, refine TableExtDesc, JoinedFlatTable,
CubeJoinedFlatTableDesc
Posted by li...@apache.org.
minior, refine TableExtDesc,JoinedFlatTable,CubeJoinedFlatTableDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0762c4dd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0762c4dd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0762c4dd
Branch: refs/heads/master-hadoop3.0
Commit: 0762c4ddbc27be4f471882394e580d94bc836fb5
Parents: a862603
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu Apr 20 15:09:33 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Tue Apr 25 16:13:53 2017 +0800
----------------------------------------------------------------------
.../cube/model/CubeJoinedFlatTableDesc.java | 2 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 8 +++-
.../kylin/metadata/model/TableExtDesc.java | 44 +++++++++++++++++---
3 files changed, 46 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index f607466..c49d37a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -44,7 +44,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Se
private int columnCount = 0;
private List<TblColRef> columnList = Lists.newArrayList();
- private Map<TblColRef, Integer> columnIndexMap = Maps.newHashMap();;
+ private Map<TblColRef, Integer> columnIndexMap = Maps.newHashMap();
public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) {
this(cubeDesc, null);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 54c1e61..a9da822 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -107,7 +107,13 @@ public class JoinedFlatTable {
}
public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
- final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
+ CubeSegment segment = ((CubeSegment) flatDesc.getSegment());
+ KylinConfig kylinConfig;
+ if (null == segment) {
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ } else {
+ kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
+ }
if (kylinConfig.isAdvancedFlatTableUsed()) {
try {
Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 18fefc2..9692f5a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -49,9 +49,11 @@ public class TableExtDesc extends RootPersistentEntity {
private List<String[]> sampleRows = new ArrayList<>();
@JsonProperty("last_modified_time")
- private String lastModifiedTime;
+ private long lastModifiedTime;
@JsonProperty("total_rows")
- private String totalRows;
+ private long totalRows;
+ @JsonProperty("mapper_rows")
+ private List<Long> mapRecords = new ArrayList<>();
@JsonProperty("data_source_properties")
private Map<String, String> dataSourceProps = new HashMap<>();
@@ -90,6 +92,14 @@ public class TableExtDesc extends RootPersistentEntity {
return this.sampleRows;
}
+ public void setMapRecords(List<Long> mapRecords) {
+ this.mapRecords = mapRecords;
+ }
+
+ public List<Long> getMapRecords() {
+ return this.mapRecords;
+ }
+
public String getCardinality() {
StringBuffer cardinality = new StringBuffer();
@@ -131,11 +141,11 @@ public class TableExtDesc extends RootPersistentEntity {
this.columnStats = columnStats;
}
- public void setTotalRows(String totalRows) {
+ public void setTotalRows(long totalRows) {
this.totalRows = totalRows;
}
- public String getTotalRows() {
+ public long getTotalRows() {
return this.totalRows;
}
@@ -152,11 +162,11 @@ public class TableExtDesc extends RootPersistentEntity {
this.tableName = this.tableName.toUpperCase();
}
- public void setLastModifiedTime(String lastModifiedTime) {
+ public void setLastModifiedTime(long lastModifiedTime) {
this.lastModifiedTime = lastModifiedTime;
}
- public String getLastModifiedTime() {
+ public long getLastModifiedTime() {
return this.lastModifiedTime;
}
@@ -187,6 +197,9 @@ public class TableExtDesc extends RootPersistentEntity {
@JsonBackReference
private TableExtDesc tableExtDesc;
+ @JsonProperty("column_name")
+ private String columnName;
+
@JsonProperty("max_value")
private String maxValue;
@@ -205,6 +218,9 @@ public class TableExtDesc extends RootPersistentEntity {
@JsonProperty("cardinality")
private long cardinality;
+ @JsonProperty("data_skew_samples")
+ private Map<String, Long> dataSkewSamples = new HashMap<>();
+
@Override
public int compareTo(ColumnStats o) {
return 0;
@@ -213,6 +229,14 @@ public class TableExtDesc extends RootPersistentEntity {
public ColumnStats() {
}
+ public void setColumnName(String columnName) {
+ this.columnName = columnName;
+ }
+
+ public String getColumnName() {
+ return this.columnName;
+ }
+
public void setMaxValue(String maxValue) {
this.maxValue = maxValue;
}
@@ -253,6 +277,14 @@ public class TableExtDesc extends RootPersistentEntity {
return this.cardinality;
}
+ public void setDataSkewSamples(Map<String, Long> dataSkewSamples) {
+ this.dataSkewSamples = dataSkewSamples;
+ }
+
+ public Map<String, Long> getDataSkewSamples() {
+ return this.dataSkewSamples;
+ }
+
public void setColumnSamples(String max, String min, String maxLenValue, String minLenValue) {
this.maxValue = max;
this.minValue = min;
[14/14] kylin git commit: KYLIN-2565, upgrade to Hadoop3.0
Posted by li...@apache.org.
KYLIN-2565, upgrade to Hadoop3.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dcb7cf47
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dcb7cf47
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dcb7cf47
Branch: refs/heads/master-hadoop3.0
Commit: dcb7cf47ce97d7e65a1327cf803a520e976f95ad
Parents: a0b537b
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Apr 25 18:45:57 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Wed Apr 26 15:45:56 2017 +0800
----------------------------------------------------------------------
.../common/DefaultSslProtocolSocketFactory.java | 150 -------------------
.../engine/mr/common/HadoopStatusGetter.java | 70 ++++++---
.../kylin/engine/spark/SparkCountDemo.java | 4 +-
.../apache/kylin/engine/spark/SparkCubing.java | 22 +--
.../kylin/engine/spark/SparkCubingByLayer.java | 25 ++--
pom.xml | 21 ++-
server-base/pom.xml | 5 +
.../apache/kylin/rest/security/MockHTable.java | 116 ++++++++++----
.../apache/kylin/rest/service/AclService.java | 2 +-
.../apache/kylin/rest/service/AdminService.java | 12 +-
storage-hbase/pom.xml | 4 +
.../kylin/storage/hbase/HBaseConnection.java | 5 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 2 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../kylin/storage/hbase/steps/CubeHFileJob.java | 18 ++-
.../storage/hbase/steps/HBaseCuboidWriter.java | 2 +-
.../storage/hbase/util/CubeMigrationCLI.java | 2 +-
.../hbase/util/DeployCoprocessorCLI.java | 5 +-
.../hbase/util/ExtendCubeToHybridCLI.java | 2 +-
.../hbase/util/GridTableHBaseBenchmark.java | 2 +-
.../kylin/storage/hbase/util/PingHBaseCLI.java | 3 +-
.../hbase/steps/CubeHFileMapperTest.java | 22 ++-
.../storage/hbase/steps/TestHbaseClient.java | 14 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 25 ++--
.../kylin/tool/CubeMigrationCheckCLI.java | 17 ++-
.../kylin/tool/ExtendCubeToHybridCLI.java | 2 +-
.../apache/kylin/tool/StorageCleanupJob.java | 18 ++-
.../org/apache/kylin/tool/util/ToolUtil.java | 9 +-
29 files changed, 293 insertions(+), 294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DefaultSslProtocolSocketFactory.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DefaultSslProtocolSocketFactory.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DefaultSslProtocolSocketFactory.java
deleted file mode 100644
index d66e4eb..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/DefaultSslProtocolSocketFactory.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.common;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.Socket;
-import java.net.UnknownHostException;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-
-import org.apache.commons.httpclient.ConnectTimeoutException;
-import org.apache.commons.httpclient.HttpClientError;
-import org.apache.commons.httpclient.params.HttpConnectionParams;
-import org.apache.commons.httpclient.protocol.ControllerThreadSocketFactory;
-import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xduo
- *
- */
-public class DefaultSslProtocolSocketFactory implements SecureProtocolSocketFactory {
- /** Log object for this class. */
- private static Logger logger = LoggerFactory.getLogger(DefaultSslProtocolSocketFactory.class);
- private SSLContext sslcontext = null;
-
- /**
- * Constructor for DefaultSslProtocolSocketFactory.
- */
- public DefaultSslProtocolSocketFactory() {
- super();
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int,java.net.InetAddress,int)
- */
- public Socket createSocket(String host, int port, InetAddress clientHost, int clientPort) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(host, port, clientHost, clientPort);
- }
-
- /**
- * Attempts to get a new socket connection to the given host within the
- * given time limit.
- *
- * <p>
- * To circumvent the limitations of older JREs that do not support connect
- * timeout a controller thread is executed. The controller thread attempts
- * to create a new socket within the given limit of time. If socket
- * constructor does not return until the timeout expires, the controller
- * terminates and throws an {@link ConnectTimeoutException}
- * </p>
- *
- * @param host
- * the host name/IP
- * @param port
- * the port on the host
- * @param localAddress
- * the local host name/IP to bind the socket to
- * @param localPort
- * the port on the local machine
- * @param params
- * {@link HttpConnectionParams Http connection parameters}
- *
- * @return Socket a new socket
- *
- * @throws IOException
- * if an I/O error occurs while creating the socket
- * @throws UnknownHostException
- * if the IP address of the host cannot be determined
- * @throws ConnectTimeoutException
- * DOCUMENT ME!
- * @throws IllegalArgumentException
- * DOCUMENT ME!
- */
- public Socket createSocket(final String host, final int port, final InetAddress localAddress, final int localPort, final HttpConnectionParams params) throws IOException, UnknownHostException, ConnectTimeoutException {
- if (params == null) {
- throw new IllegalArgumentException("Parameters may not be null");
- }
-
- int timeout = params.getConnectionTimeout();
-
- if (timeout == 0) {
- return createSocket(host, port, localAddress, localPort);
- } else {
- // To be eventually deprecated when migrated to Java 1.4 or above
- return ControllerThreadSocketFactory.createSocket(this, host, port, localAddress, localPort, timeout);
- }
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int)
- */
- public Socket createSocket(String host, int port) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(host, port);
- }
-
- /**
- * @see SecureProtocolSocketFactory#createSocket(java.net.Socket,java.lang.String,int,boolean)
- */
- public Socket createSocket(Socket socket, String host, int port, boolean autoClose) throws IOException, UnknownHostException {
- return getSSLContext().getSocketFactory().createSocket(socket, host, port, autoClose);
- }
-
- public boolean equals(Object obj) {
- return ((obj != null) && obj.getClass().equals(DefaultX509TrustManager.class));
- }
-
- public int hashCode() {
- return DefaultX509TrustManager.class.hashCode();
- }
-
- private static SSLContext createEasySSLContext() {
- try {
- SSLContext context = SSLContext.getInstance("TLS");
- context.init(null, new TrustManager[] { new DefaultX509TrustManager(null) }, null);
-
- return context;
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new HttpClientError(e.toString());
- }
- }
-
- private SSLContext getSSLContext() {
- if (this.sslcontext == null) {
- this.sslcontext = createEasySSLContext();
- }
-
- return this.sslcontext;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index f31369b..0245c1c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -21,25 +21,31 @@ package org.apache.kylin.engine.mr.common;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.charset.Charset;
+import java.security.KeyManagementException;
import java.security.Principal;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.protocol.Protocol;
-import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthSchemeRegistry;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
+import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.AuthPolicy;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultHttpClient;
@@ -107,7 +113,7 @@ public class HadoopStatusGetter {
String response = null;
while (response == null) {
if (url.startsWith("https://")) {
- registerEasyHttps();
+ registerEasyHttps(client);
}
if (url.contains("anonymous=true") == false) {
url += url.contains("?") ? "&" : "?";
@@ -162,26 +168,26 @@ public class HadoopStatusGetter {
}
private String getHttpResponse(String url) throws IOException {
- HttpClient client = new HttpClient();
+ HttpClient client = new DefaultHttpClient();
String response = null;
while (response == null) { // follow redirects via 'refresh'
if (url.startsWith("https://")) {
- registerEasyHttps();
+ registerEasyHttps(client);
}
if (url.contains("anonymous=true") == false) {
url += url.contains("?") ? "&" : "?";
url += "anonymous=true";
}
- HttpMethod get = new GetMethod(url);
- get.addRequestHeader("accept", "application/json");
+ HttpGet get = new HttpGet(url);
+ get.addHeader("accept", "application/json");
try {
- client.executeMethod(get);
+ HttpResponse res = client.execute(get);
String redirect = null;
- Header h = get.getResponseHeader("Location");
+ Header h = res.getFirstHeader("Location");
if (h != null) {
redirect = h.getValue();
if (isValidURL(redirect) == false) {
@@ -190,7 +196,7 @@ public class HadoopStatusGetter {
continue;
}
} else {
- h = get.getResponseHeader("Refresh");
+ h = res.getFirstHeader("Refresh");
if (h != null) {
String s = h.getValue();
int cut = s.indexOf("url=");
@@ -207,7 +213,7 @@ public class HadoopStatusGetter {
}
if (redirect == null) {
- response = get.getResponseBodyAsString();
+ response = res.getStatusLine().toString();
logger.debug("Job " + mrJobId + " get status check result.\n");
} else {
url = redirect;
@@ -224,13 +230,35 @@ public class HadoopStatusGetter {
return response;
}
- private static Protocol EASY_HTTPS = null;
+ private static void registerEasyHttps(HttpClient client) {
+ SSLContext sslContext;
+ try {
+ sslContext = SSLContext.getInstance("SSL");
+
+ // set up a TrustManager that trusts everything
+ try {
+ sslContext.init(null, new TrustManager[] { new DefaultX509TrustManager(null) {
+ public X509Certificate[] getAcceptedIssuers() {
+ logger.debug("getAcceptedIssuers");
+ return null;
+ }
+
+ public void checkClientTrusted(X509Certificate[] certs, String authType) {
+ logger.debug("checkClientTrusted");
+ }
- private static void registerEasyHttps() {
- // by pass all https issue
- if (EASY_HTTPS == null) {
- EASY_HTTPS = new Protocol("https", (ProtocolSocketFactory) new DefaultSslProtocolSocketFactory(), 443);
- Protocol.registerProtocol("https", EASY_HTTPS);
+ public void checkServerTrusted(X509Certificate[] certs, String authType) {
+ logger.debug("checkServerTrusted");
+ }
+ } }, new SecureRandom());
+ } catch (KeyManagementException e) {
+ }
+ SSLSocketFactory ssf = new SSLSocketFactory(sslContext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ ClientConnectionManager ccm = client.getConnectionManager();
+ SchemeRegistry sr = ccm.getSchemeRegistry();
+ sr.register(new Scheme("https", 443, ssf));
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
index 6478c10..a079a57 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
@@ -22,7 +22,7 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.spark.SparkConf;
@@ -74,7 +74,7 @@ public class SparkCountDemo extends AbstractApplication {
KeyValue value = new KeyValue(stringIntegerTuple2._1().getBytes(), "f".getBytes(), "c".getBytes(), String.valueOf(stringIntegerTuple2._2()).getBytes());
return new Tuple2(key, value);
}
- }).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class);
+ }).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat2.class);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 2a0981a..a87d66b 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -41,9 +41,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
@@ -367,7 +369,7 @@ public class SparkCubing extends AbstractApplication {
final JavaPairRDD<byte[], byte[]> javaPairRDD = javaRDD.glom().mapPartitionsToPair(new PairFlatMapFunction<Iterator<List<List<String>>>, byte[], byte[]>() {
@Override
- public Iterable<Tuple2<byte[], byte[]>> call(Iterator<List<List<String>>> listIterator) throws Exception {
+ public Iterator<Tuple2<byte[], byte[]>> call(Iterator<List<List<String>>> listIterator) throws Exception {
long t = System.currentTimeMillis();
prepare();
@@ -390,7 +392,7 @@ public class SparkCubing extends AbstractApplication {
throw new RuntimeException(e);
}
System.out.println("build partition cost: " + (System.currentTimeMillis() - t) + "ms");
- return sparkCuboidWriter.getResult();
+ return sparkCuboidWriter.getResult().iterator();
}
});
@@ -430,8 +432,8 @@ public class SparkCubing extends AbstractApplication {
}
}, UnsignedBytes.lexicographicalComparator()).mapPartitions(new FlatMapFunction<Iterator<Tuple2<byte[], byte[]>>, Tuple2<byte[], byte[]>>() {
@Override
- public Iterable<Tuple2<byte[], byte[]>> call(final Iterator<Tuple2<byte[], byte[]>> tuple2Iterator) throws Exception {
- return new Iterable<Tuple2<byte[], byte[]>>() {
+ public Iterator<Tuple2<byte[], byte[]>> call(final Iterator<Tuple2<byte[], byte[]>> tuple2Iterator) throws Exception {
+ Iterable<Tuple2<byte[], byte[]>> iterable = new Iterable<Tuple2<byte[], byte[]>>() {
final BufferedMeasureCodec codec = new BufferedMeasureCodec(dataTypes);
final Object[] input = new Object[measureSize];
final Object[] result = new Object[measureSize];
@@ -459,6 +461,7 @@ public class SparkCubing extends AbstractApplication {
});
}
};
+ return iterable.iterator();
}
}, true).mapToPair(new PairFunction<Tuple2<byte[], byte[]>, ImmutableBytesWritable, KeyValue>() {
@Override
@@ -467,7 +470,7 @@ public class SparkCubing extends AbstractApplication {
KeyValue value = new KeyValue(tuple2._1(), "F1".getBytes(), "M".getBytes(), tuple2._2());
return new Tuple2(key, value);
}
- }).saveAsNewAPIHadoopFile(hFileLocation, ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class, conf);
+ }).saveAsNewAPIHadoopFile(hFileLocation, ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat2.class, conf);
}
public static void prepare() throws Exception {
@@ -496,8 +499,9 @@ public class SparkCubing extends AbstractApplication {
Job job = Job.getInstance(conf);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(KeyValue.class);
- HTable table = new HTable(conf, hTableName);
- HFileOutputFormat.configureIncrementalLoad(job, table);
+ Connection connection = HBaseConnection.get();
+ Table table = connection.getTable(TableName.valueOf(hTableName));
+ HFileOutputFormat2.configureIncrementalLoad(job, table, connection.getRegionLocator(TableName.valueOf(hTableName)));
return conf;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index f70fd30..28591ef 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -17,6 +17,15 @@
*/
package org.apache.kylin.engine.spark;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -65,16 +74,8 @@ import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.storage.StorageLevel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+import scala.Tuple2;
/**
* Spark application to build cube with the "by-layer" algorithm. Only support source data from Hive; Metadata in HBase.
@@ -354,7 +355,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
@Override
- public Iterable<Tuple2<ByteArray, Object[]>> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
+ public Iterator<Tuple2<ByteArray, Object[]>> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
if (initialized == false) {
prepare();
initialized = true;
@@ -368,7 +369,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {
- return EMTPY_ITERATOR;
+ return EMTPY_ITERATOR.iterator();
}
List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size());
@@ -382,7 +383,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
tuples.add(new Tuple2<>(new ByteArray(newKey), tuple2._2()));
}
- return tuples;
+ return tuples.iterator();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0e772ca..ea35550 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,29 +41,30 @@
<properties>
<!-- General Properties -->
- <javaVersion>1.7</javaVersion>
+ <javaVersion>1.8</javaVersion>
<maven-model.version>3.3.9</maven-model.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>3.0.0-alpha2</hadoop2.version>
+ <yarn.version>3.0.0-alpha2</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>2.1.0</hive.version>
+ <hive-hcatalog.version>2.1.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>2.0.0-SNAPSHOT</hbase-hadoop2.version>
<!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
<!-- Spark versions -->
- <spark.version>1.6.3</spark.version>
+ <spark.version>2.0.0-SNAPSHOT</spark.version>
<kryo.version>4.0.0</kryo.version>
+ <commons-configuration.version>1.6</commons-configuration.version>
<!-- <reflections.version>0.9.10</reflections.version> -->
<!-- Calcite Version -->
@@ -518,6 +519,12 @@
<version>${yarn.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>${commons-configuration.version}</version>
+ </dependency>
+
<!-- Calcite dependencies -->
<dependency>
<groupId>org.apache.calcite</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index 627ae35..34d8ffb 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -169,6 +169,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<repositories>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index 972eea9..fd53b5b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -43,6 +43,8 @@ import java.util.TreeMap;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
@@ -51,7 +53,6 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -60,7 +61,9 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
@@ -97,7 +100,7 @@ public class MockHTable implements Table {
private NavigableMap<byte[], NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>> data = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- private static List<KeyValue> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, int maxVersions) {
+ private static List<Cell> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, int maxVersions) {
return toKeyValue(row, rowdata, 0, Long.MAX_VALUE, maxVersions);
}
@@ -162,8 +165,8 @@ public class MockHTable implements Table {
throw new RuntimeException(this.getClass() + " does NOT implement this method.");
}
- private static List<KeyValue> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, long timestampStart, long timestampEnd, int maxVersions) {
- List<KeyValue> ret = new ArrayList<KeyValue>();
+ private static List<Cell> toKeyValue(byte[] row, NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowdata, long timestampStart, long timestampEnd, int maxVersions) {
+ List<Cell> ret = new ArrayList<>();
for (byte[] family : rowdata.keySet())
for (byte[] qualifier : rowdata.get(family).keySet()) {
int versionsAdded = 0;
@@ -207,7 +210,6 @@ public class MockHTable implements Table {
/**
* {@inheritDoc}
*/
- @Override
public Object[] batch(List<? extends Row> actions) throws IOException, InterruptedException {
Object[] results = new Object[actions.size()]; // same size.
for (int i = 0; i < actions.size(); i++) {
@@ -241,11 +243,6 @@ public class MockHTable implements Table {
}
- @Override
- public <R> Object[] batchCallback(List<? extends Row> actions, Batch.Callback<R> callback) throws IOException, InterruptedException {
- return new Object[0];
- }
-
/**
* {@inheritDoc}
*/
@@ -254,7 +251,7 @@ public class MockHTable implements Table {
if (!data.containsKey(get.getRow()))
return new Result();
byte[] row = get.getRow();
- List<KeyValue> kvs = new ArrayList<KeyValue>();
+ List<Cell> kvs = new ArrayList<>();
if (!get.hasFamilies()) {
kvs = toKeyValue(row, data.get(row), get.getMaxVersions());
} else {
@@ -279,7 +276,7 @@ public class MockHTable implements Table {
kvs = filter(filter, kvs);
}
- return new Result(kvs);
+ return Result.create(kvs);
}
/**
@@ -317,11 +314,11 @@ public class MockHTable implements Table {
break;
}
- List<KeyValue> kvs = null;
+ List<Cell> kvs = null;
if (!scan.hasFamilies()) {
kvs = toKeyValue(row, data.get(row), scan.getTimeRange().getMin(), scan.getTimeRange().getMax(), scan.getMaxVersions());
} else {
- kvs = new ArrayList<KeyValue>();
+ kvs = new ArrayList<>();
for (byte[] family : scan.getFamilyMap().keySet()) {
if (data.get(row).get(family) == null)
continue;
@@ -353,7 +350,7 @@ public class MockHTable implements Table {
}
}
if (!kvs.isEmpty()) {
- ret.add(new Result(kvs));
+ ret.add(Result.create(kvs));
}
}
@@ -387,6 +384,16 @@ public class MockHTable implements Table {
public void close() {
}
+
+ @Override
+ public boolean renewLease() {
+ return false;
+ }
+
+ @Override
+ public ScanMetrics getScanMetrics() {
+ return null;
+ }
};
}
@@ -397,10 +404,10 @@ public class MockHTable implements Table {
* @param kvs List of a row's KeyValues
* @return List of KeyValues that were not filtered.
*/
- private List<KeyValue> filter(Filter filter, List<KeyValue> kvs) throws IOException {
+ private List<Cell> filter(Filter filter, List<Cell> kvs) throws IOException {
filter.reset();
- List<KeyValue> tmp = new ArrayList<KeyValue>(kvs.size());
+ List<Cell> tmp = new ArrayList<>(kvs.size());
tmp.addAll(kvs);
/*
@@ -409,9 +416,9 @@ public class MockHTable implements Table {
* See Figure 4-2 on p. 163.
*/
boolean filteredOnRowKey = false;
- List<KeyValue> nkvs = new ArrayList<KeyValue>(tmp.size());
- for (KeyValue kv : tmp) {
- if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
+ List<Cell> nkvs = new ArrayList<>(tmp.size());
+ for (Cell kv : tmp) {
+ if (filter.filterRowKey(kv)) {
filteredOnRowKey = true;
break;
}
@@ -474,16 +481,16 @@ public class MockHTable implements Table {
public void put(Put put) throws IOException {
byte[] row = put.getRow();
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR));
- for (byte[] family : put.getFamilyMap().keySet()) {
+ for (byte[] family : put.getFamilyCellMap().keySet()) {
if (columnFamilies.contains(new String(family)) == false) {
throw new RuntimeException("Not Exists columnFamily : " + new String(family));
}
NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR));
- for (KeyValue kv : put.getFamilyMap().get(family)) {
- kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis()));
- byte[] qualifier = kv.getQualifier();
+ for (Cell kv : put.getFamilyCellMap().get(family)) {
+ CellUtil.updateLatestStamp(kv, System.currentTimeMillis());
+ byte[] qualifier = kv.getQualifierArray();
NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>());
- qualifierData.put(kv.getTimestamp(), kv.getValue());
+ qualifierData.put(kv.getTimestamp(), kv.getValueArray());
}
}
}
@@ -531,22 +538,22 @@ public class MockHTable implements Table {
byte[] row = delete.getRow();
if (data.get(row) == null)
return;
- if (delete.getFamilyMap().size() == 0) {
+ if (delete.getFamilyCellMap().size() == 0) {
data.remove(row);
return;
}
- for (byte[] family : delete.getFamilyMap().keySet()) {
+ for (byte[] family : delete.getFamilyCellMap().keySet()) {
if (data.get(row).get(family) == null)
continue;
- if (delete.getFamilyMap().get(family).isEmpty()) {
+ if (delete.getFamilyCellMap().get(family).isEmpty()) {
data.get(row).remove(family);
continue;
}
- for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDelete()) {
- data.get(row).get(kv.getFamily()).clear();
+ for (Cell kv : delete.getFamilyCellMap().get(family)) {
+ if (CellUtil.isDelete(kv)) {
+ data.get(row).get(kv.getFamilyArray()).clear();
} else {
- data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
+ data.get(row).get(kv.getFamilyArray()).remove(kv.getQualifierArray());
}
}
if (data.get(row).get(family).isEmpty()) {
@@ -665,4 +672,49 @@ public class MockHTable implements Table {
throw new NotImplementedException();
}
+
+ /***
+ *
+ * All values are default
+ *
+ * **/
+ @Override
+ public void setOperationTimeout(int i) {
+
+ }
+
+ @Override
+ public int getOperationTimeout() {
+ return 0;
+ }
+
+ @Override
+ public int getRpcTimeout() {
+ return 0;
+ }
+
+ @Override
+ public void setRpcTimeout(int i) {
+
+ }
+
+ @Override
+ public int getReadRpcTimeout() {
+ return 0;
+ }
+
+ @Override
+ public void setReadRpcTimeout(int i) {
+
+ }
+
+ @Override
+ public int getWriteRpcTimeout() {
+ return 0;
+ }
+
+ @Override
+ public void setWriteRpcTimeout(int i) {
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index b80d97d..184e4b0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -290,7 +290,7 @@ public class AclService implements MutableAclService {
htable = aclHBaseStorage.getTable(aclTableName);
Delete delete = new Delete(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
- delete.deleteFamily(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY));
+ delete.addFamily(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY));
htable.delete(delete);
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
index 66725dc..e06157d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java
@@ -18,12 +18,6 @@
package org.apache.kylin.rest.service;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.kylin.common.KylinConfig;
@@ -36,6 +30,12 @@ import org.slf4j.LoggerFactory;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
/**
* @author jianliu
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 29ca7e5..118a946 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -96,6 +96,10 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 7e2cefc..714c669 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -210,6 +210,11 @@ public class HBaseConnection {
// ============================================================================
+ public static Connection get() {
+ String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
+ return get(url);
+ }
+
// returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
public static Connection get(String url) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index e822ada..aad5234 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
-import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
@@ -183,7 +183,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
new Batch.Call<CubeVisitService, CubeVisitResponse>() {
public CubeVisitResponse call(CubeVisitService rowsService) throws IOException {
ServerRpcController controller = new ServerRpcController();
- BlockingRpcCallback<CubeVisitResponse> rpcCallback = new BlockingRpcCallback<>();
+ CoprocessorRpcUtils.BlockingRpcCallback<CubeVisitResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<>();
rowsService.visitCube(controller, request, rpcCallback);
CubeVisitResponse response = rpcCallback.get();
if (controller.failedOnException()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 951e2ef..e116935 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -181,7 +181,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
public List<Cell> next() {
List<Cell> result = allResultsIterator.next().listCells();
for (Cell cell : result) {
- scannedBytes += CellUtil.estimatedSizeOf(cell);
+ scannedBytes += CellUtil.estimatedSerializedSizeOf(cell);
}
scannedRows++;
return result;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index cde127e..6142025 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -37,9 +37,9 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
import org.apache.kylin.common.exceptions.ResourceLimitExceededException;
@@ -177,7 +177,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
List<Cell> result = delegate.next();
rowCount++;
for (Cell cell : result) {
- rowBytes += CellUtil.estimatedSizeOf(cell);
+ rowBytes += CellUtil.estimatedSerializedSizeOf(cell);
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
index 1a624c4..0b92c64 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -25,8 +25,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.SequenceFile;
@@ -56,6 +60,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
public int run(String[] args) throws Exception {
Options options = new Options();
+ Connection connection = null;
try {
options.addOption(OPTION_JOB_NAME);
options.addOption(OPTION_CUBE_NAME);
@@ -92,10 +97,13 @@ public class CubeHFileJob extends AbstractHadoopJob {
attachCubeMetadata(cube, job.getConfiguration());
Configuration hbaseConf = HBaseConfiguration.create(getConf());
- HTable htable = new HTable(hbaseConf, getOptionValue(OPTION_HTABLE_NAME).toUpperCase());
+ String hTableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
+ connection = ConnectionFactory.createConnection(hbaseConf);
+ Table table = connection.getTable(TableName.valueOf(hTableName));
+ RegionLocator regionLocator = connection.getRegionLocator(TableName.valueOf(hTableName));
// Automatic config !
- HFileOutputFormat.configureIncrementalLoad(job, htable);
+ HFileOutputFormat2.configureIncrementalLoad(job, table, regionLocator);
reconfigurePartitions(hbaseConf, partitionFilePath);
// set block replication to 3 for hfiles
@@ -107,6 +115,8 @@ public class CubeHFileJob extends AbstractHadoopJob {
} finally {
if (job != null)
cleanupTempConfFile(job.getConfiguration());
+ if (null != connection)
+ connection.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 6587d4e..7bb5ecd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -103,7 +103,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
byte[] family = copy(keyValue.getFamilyArray(), keyValue.getFamilyOffset(), keyValue.getFamilyLength());
byte[] qualifier = copy(keyValue.getQualifierArray(), keyValue.getQualifierOffset(), keyValue.getQualifierLength());
byte[] value = copy(keyValue.getValueArray(), keyValue.getValueOffset(), keyValue.getValueLength());
- put.add(family, qualifier, value);
+ put.addColumn(family, qualifier, value);
puts.add(put);
}
if (puts.size() >= BATCH_PUT_THRESHOLD) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 581de38..58fba8d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -470,7 +470,7 @@ public class CubeMigrationCLI {
value = Bytes.toBytes(valueString);
}
Put put = new Put(Bytes.toBytes(cubeId));
- put.add(family, column, value);
+ put.addColumn(family, column, value);
destAclHtable.put(put);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index d51b71e..cee0aa9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.Bytes;
@@ -156,7 +155,7 @@ public class DeployCoprocessorCLI {
ProjectInstance projectInstance = projectManager.getProject(p);
List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
- for (RealizationEntry cube: cubeList) {
+ for (RealizationEntry cube : cubeList) {
CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
for (CubeSegment segment : cubeInstance.getSegments()) {
String tableName = segment.getStorageLocationIdentifier();
@@ -440,7 +439,7 @@ public class DeployCoprocessorCLI {
Matcher keyMatcher;
Matcher valueMatcher;
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e : tableDescriptor.getValues().entrySet()) {
+ for (Map.Entry<org.apache.hadoop.hbase.util.Bytes, org.apache.hadoop.hbase.util.Bytes> e : tableDescriptor.getValues().entrySet()) {
keyMatcher = HConstants.CP_HTD_ATTR_KEY_PATTERN.matcher(Bytes.toString(e.getKey().get()));
if (!keyMatcher.matches()) {
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 1cdb2f8..18b894a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -254,7 +254,7 @@ public class ExtendCubeToHybridCLI {
value = Bytes.toBytes(valueString);
}
Put put = new Put(Bytes.toBytes(newCubeId));
- put.add(family, column, value);
+ put.addColumn(family, column, value);
aclHtable.put(put);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index dd5f8fa..218841c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -232,7 +232,7 @@ public class GridTableHBaseBenchmark {
byte[] rowkey = Bytes.toBytes(i);
Put put = new Put(rowkey);
byte[] cell = randomBytes();
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
table.put(put);
nBytes += cell.length;
dot(i, N_ROWS);
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index bba6745..ff038d1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -50,7 +50,8 @@ public class PingHBaseCLI {
if (User.isHBaseSecurityEnabled(hconf)) {
try {
System.out.println("--------------Getting kerberos credential for user " + UserGroupInformation.getCurrentUser().getUserName());
- TokenUtil.obtainAndCacheToken(hconf, UserGroupInformation.getCurrentUser());
+ Connection connection = HBaseConnection.get();
+ TokenUtil.obtainAndCacheToken(connection, User.create(UserGroupInformation.getCurrentUser()));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
System.out.println("--------------Error while getting kerberos credential for user " + UserGroupInformation.getCurrentUser().getUserName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapperTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapperTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapperTest.java
index f8282d3..d019188 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapperTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapperTest.java
@@ -68,13 +68,23 @@ public class CubeHFileMapperTest {
Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1);
assertEquals(key, p1.getFirst());
- assertEquals("cf1", new String(p1.getSecond().getFamily()));
- assertEquals("usd_amt", new String(p1.getSecond().getQualifier()));
- assertEquals("35.43", new String(p1.getSecond().getValue()));
+ assertEquals("cf1", new String(copy(p1.getSecond())));
+ assertEquals("usd_amt", new String(copy(p1.getSecond())));
+ assertEquals("35.43", new String(copy(p1.getSecond())));
assertEquals(key, p2.getFirst());
- assertEquals("cf1", new String(p2.getSecond().getFamily()));
- assertEquals("item_count", new String(p2.getSecond().getQualifier()));
- assertEquals("2", new String(p2.getSecond().getValue()));
+ assertEquals("cf1", new String(copy(p2.getSecond())));
+ assertEquals("item_count", new String(copy(p2.getSecond())));
+ assertEquals("2", new String(copy(p2.getSecond())));
+ }
+
+ private byte[] copy(KeyValue kv) {
+ return copy(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength());
+ }
+
+ private byte[] copy(byte[] array, int offset, int length) {
+ byte[] result = new byte[length];
+ System.arraycopy(array, offset, result, 0, length);
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/TestHbaseClient.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/TestHbaseClient.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/TestHbaseClient.java
index 2b8ecae..b77d2cb 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/TestHbaseClient.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/TestHbaseClient.java
@@ -22,8 +22,11 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
/**
@@ -89,13 +92,16 @@ public class TestHbaseClient {
conf.set("hbase.zookeeper.quorum", "hbase_host");
conf.set("zookeeper.znode.parent", "/hbase-unsecure");
- HTable table = new HTable(conf, "test1");
+ Connection connection = ConnectionFactory.createConnection(conf);
+
+ Table table = connection.getTable(TableName.valueOf("test1"));
Put put = new Put(Bytes.toBytes("row1"));
- put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
- put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val2"));
+ put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
+ put.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val2"));
table.put(put);
table.close();
+ connection.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index c162a76..eb4f492 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -26,16 +26,16 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
@@ -90,7 +90,8 @@ public class CubeMigrationCLI {
private ResourceStore srcStore;
private ResourceStore dstStore;
private FileSystem hdfsFS;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
+ private Connection connection;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -134,8 +135,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ connection = HBaseConnection.get();
+ hbaseAdmin = connection.getAdmin();
hdfsFS = HadoopUtil.getWorkingFileSystem();
@@ -337,10 +338,10 @@ public class CubeMigrationCLI {
String tableName = (String) opt.params[0];
System.out.println("CHANGE_HTABLE_HOST, table name: " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
logger.info("CHANGE_HTABLE_HOST is completed");
break;
}
@@ -482,7 +483,7 @@ public class CubeMigrationCLI {
value = Bytes.toBytes(valueString);
}
Put put = new Put(Bytes.toBytes(cubeId));
- put.add(family, column, value);
+ put.addColumn(family, column, value);
destAclHtable.put(put);
}
}
@@ -518,10 +519,10 @@ public class CubeMigrationCLI {
case CHANGE_HTABLE_HOST: {
String tableName = (String) opt.params[0];
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
break;
}
case COPY_FILE_IN_META: {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
index 54fbbc0..52bad9d 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCheckCLI.java
@@ -29,7 +29,9 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +63,8 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
+ private Connection connection;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -123,6 +126,7 @@ public class CubeMigrationCheckCLI {
}
fixInconsistent();
printIssueExistingHTables();
+ connection.close();
}
public CubeMigrationCheckCLI(KylinConfig kylinConfig, Boolean isFix) throws IOException {
@@ -130,7 +134,8 @@ public class CubeMigrationCheckCLI {
this.ifFix = isFix;
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ connection = ConnectionFactory.createConnection(conf);
+ hbaseAdmin = connection.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
@@ -189,10 +194,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..d628ca2 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -250,7 +250,7 @@ public class ExtendCubeToHybridCLI {
value = Bytes.toBytes(valueString);
}
Put put = new Put(Bytes.toBytes(newCubeId));
- put.add(family, column, value);
+ put.addColumn(family, column, value);
aclHtable.put(put);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index f1a3ebe..a725f23 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -41,7 +41,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -82,7 +85,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) {
+ Connection connection = ConnectionFactory.createConnection(conf);
+ try (Admin hbaseAdmin = connection.getAdmin()) {
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -129,6 +133,8 @@ public class StorageCleanupJob extends AbstractApplication {
}
System.out.println("----------------------------------------------------");
}
+ } finally {
+ connection.close();
}
}
@@ -157,12 +163,12 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
- String htableName;
+ Admin hbaseAdmin;
+ TableName htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
- this.htableName = htableName;
+ this.htableName = TableName.valueOf(htableName);
}
public Object call() throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/dcb7cf47/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
index c41d6a8..7eae46c 100644
--- a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
+++ b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
@@ -29,7 +29,9 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -51,12 +53,15 @@ public class ToolUtil {
}
public static String getHBaseMetaStoreId() throws IOException {
- try (final HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()))) {
+ Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()));
+ try (final Admin hbaseAdmin = connection.getAdmin()) {
final String metaStoreName = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName));
return desc.getValue(HBaseConnection.HTABLE_UUID_TAG);
} catch (Exception e) {
return null;
+ } finally {
+ connection.close();
}
}