You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/02/07 07:40:24 UTC
[kylin] branch 2.6.x updated (ae63d70 -> 9e35338)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from ae63d70 Set repository sites to https
new 1f9f44c Fix sql injection issue
new 0556b00 Prevent uncontrolled data used in path expression
new 995d5b2 Encrept response output for BroadcasterReceiveServlet
new a3ec2e6 Fix "equals" on incomparable types
new 9e35338 Fix not thread-safe double-checked locking
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/kylin/measure/topn/TopNMeasureType.java | 2 +-
.../metadata/badquery/BadQueryHistoryManager.java | 1 +
.../broadcaster/BroadcasterReceiveServlet.java | 24 ++++++++-------
.../org/apache/kylin/rest/service/CubeService.java | 36 ++++++++++++++++------
.../storage/hbase/steps/HFileOutputFormat3.java | 3 +-
5 files changed, 44 insertions(+), 22 deletions(-)
[kylin] 05/05: Fix not thread-safe double-checked locking
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9e3533865788beec74fe060d6cdc8992c6142dc4
Author: nichunen <ni...@apache.org>
AuthorDate: Fri Feb 7 15:39:08 2020 +0800
Fix not thread-safe double-checked locking
---
.../src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 472de3c..ce136ea 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -129,7 +129,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private List<TblColRef> literalCols = null;
private int keyLength = 0;
- private DimensionEncoding[] newDimensionEncodings = null;
+ private volatile DimensionEncoding[] newDimensionEncodings = null;
private int newKeyLength = 0;
private boolean needReEncode = true;
[kylin] 01/05: Fix sql injection issue
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1f9f44ceb818b46518176e81c6dea5a0d12750cf
Author: nichunen <ni...@apache.org>
AuthorDate: Fri Jan 10 21:03:31 2020 +0800
Fix sql injection issue
---
.../org/apache/kylin/rest/service/CubeService.java | 36 ++++++++++++++++------
1 file changed, 27 insertions(+), 9 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index b975cdc..c9f6fb9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -69,6 +69,7 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -77,6 +78,7 @@ import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
+import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
@@ -576,7 +578,8 @@ public class CubeService extends BasicService implements InitializingBean {
List<String> toDelHDFSPaths = Lists.newArrayListWithCapacity(toRemoveSegs.size());
for (CubeSegment seg : toRemoveSegs) {
toDropHTables.add(seg.getStorageLocationIdentifier());
- toDelHDFSPaths.add(JobBuilderSupport.getJobWorkingDir(seg.getConfig().getHdfsWorkingDirectory(), seg.getLastBuildJobID()));
+ toDelHDFSPaths.add(JobBuilderSupport.getJobWorkingDir(seg.getConfig().getHdfsWorkingDirectory(),
+ seg.getLastBuildJobID()));
}
StorageCleanUtil.dropHTables(new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration()), toDropHTables);
@@ -713,7 +716,6 @@ public class CubeService extends BasicService implements InitializingBean {
return false;
}
-
public void validateCubeDesc(CubeDesc desc, boolean isDraft) {
Message msg = MsgPicker.getMsg();
@@ -931,7 +933,7 @@ public class CubeService extends BasicService implements InitializingBean {
long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L
: queryMatchMap.get(cuboidId);
boolean ifExist = currentCuboidSet.contains(cuboidId);
- long rowCount = rowCountMap == null ? 0L : rowCountMap.get(cuboidId);
+ long rowCount = (rowCountMap == null || rowCountMap.size() == 0) ? 0L : rowCountMap.get(cuboidId);
NodeInfo node = new NodeInfo();
node.setId(cuboidId);
@@ -980,9 +982,10 @@ public class CubeService extends BasicService implements InitializingBean {
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ " from " + table//
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = ?" //
+ " group by " + cuboidColumn;
- List<List<String>> orgHitFrequency = queryService.querySystemCube(sql).getResults();
+
+ List<List<String>> orgHitFrequency = getPrepareQueryResult(cubeName, sql);
return formatQueryCount(orgHitFrequency);
}
@@ -994,9 +997,10 @@ public class CubeService extends BasicService implements InitializingBean {
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
String sql = "select " + cuboidSource + ", " + cuboidTgt + ", avg(" + aggCount + "), avg(" + returnCount + ")"//
+ " from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "' " //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = ?" //
+ " group by " + cuboidSource + ", " + cuboidTgt;
- List<List<String>> orgRollingUpCount = queryService.querySystemCube(sql).getResults();
+
+ List<List<String>> orgRollingUpCount = getPrepareQueryResult(cubeName, sql);
return formatRollingUpStats(orgRollingUpCount);
}
@@ -1006,13 +1010,27 @@ public class CubeService extends BasicService implements InitializingBean {
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQueryCube());
String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ")" //
+ " from " + table //
- + " where " + QueryCubePropertyEnum.CUBE.toString() + " = '" + cubeName + "'" //
+ + " where " + QueryCubePropertyEnum.CUBE.toString() + " = ?" //
+ " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " = true" //
+ " group by " + cuboidSource;
- List<List<String>> orgMatchHitFrequency = queryService.querySystemCube(sql).getResults();
+
+ List<List<String>> orgMatchHitFrequency = getPrepareQueryResult(cubeName, sql);
return formatQueryCount(orgMatchHitFrequency);
}
+ private List<List<String>> getPrepareQueryResult(String cubeName, String sql) {
+ PrepareSqlRequest sqlRequest = new PrepareSqlRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ PrepareSqlRequest.StateParam[] params = new PrepareSqlRequest.StateParam[1];
+ params[0] = new PrepareSqlRequest.StateParam();
+ params[0].setClassName("java.lang.String");
+ params[0].setValue(cubeName);
+ sqlRequest.setParams(params);
+ sqlRequest.setSql(sql);
+
+ return queryService.doQueryWithCache(sqlRequest, false).getResults();
+ }
+
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void migrateCube(CubeInstance cube, String projectName) {
[kylin] 04/05: Fix "equals" on incomparable types
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a3ec2e65c90db637546a74b3e827b7798020d460
Author: nichunen <ni...@apache.org>
AuthorDate: Sun Jan 12 18:42:25 2020 +0800
Fix "equals" on incomparable types
---
.../java/org/apache/kylin/storage/hbase/steps/HFileOutputFormat3.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HFileOutputFormat3.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HFileOutputFormat3.java
index 1f75660..12c30ea 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HFileOutputFormat3.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HFileOutputFormat3.java
@@ -23,6 +23,7 @@ import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -286,7 +287,7 @@ public class HFileOutputFormat3 extends FileOutputFormat<ImmutableBytesWritable,
TreeSet<ImmutableBytesWritable> sorted = new TreeSet<ImmutableBytesWritable>(startKeys);
ImmutableBytesWritable first = sorted.first();
- if (!first.equals(HConstants.EMPTY_BYTE_ARRAY)) {
+ if (!Arrays.equals(first.get(), HConstants.EMPTY_BYTE_ARRAY)) {
throw new IllegalArgumentException("First region of table should have empty start key. Instead has: "
+ Bytes.toStringBinary(first.get()));
}
[kylin] 02/05: Prevent uncontrolled data used in path expression
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0556b007dd7e9e69f6ce817a883b38f7c69c58de
Author: nichunen <ni...@apache.org>
AuthorDate: Fri Jan 10 21:04:21 2020 +0800
Prevent uncontrolled data used in path expression
---
.../java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
index 843e9e9..812d3c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
@@ -57,6 +57,7 @@ public class BadQueryHistoryManager {
}
public BadQueryHistory getBadQueriesForProject(String project) throws IOException {
+ project = project.replaceAll("[./]", "");
BadQueryHistory badQueryHistory = getStore().getResource(getResourcePathForProject(project), BAD_QUERY_INSTANCE_SERIALIZER);
if (badQueryHistory == null) {
badQueryHistory = new BadQueryHistory(project);
[kylin] 03/05: Encrept response output for BroadcasterReceiveServlet
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 995d5b2f48cc253863bc328034cdd9b9ff638023
Author: nichunen <ni...@apache.org>
AuthorDate: Fri Jan 10 21:51:03 2020 +0800
Encrept response output for BroadcasterReceiveServlet
---
.../broadcaster/BroadcasterReceiveServlet.java | 24 ++++++++++++----------
1 file changed, 13 insertions(+), 11 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java b/server-base/src/main/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
index a277cf3..8450f7a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/broadcaster/BroadcasterReceiveServlet.java
@@ -28,26 +28,21 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.kylin.common.util.EncryptUtil;
+
/**
*/
public class BroadcasterReceiveServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
-
- public interface BroadcasterHandler {
-
- void handle(String type, String name, String event);
- }
-
+ private static final Pattern PATTERN = Pattern.compile("/(.+)/(.+)/(.+)");
+ private static final Pattern PATTERN2 = Pattern.compile("/(.+)/(.+)");
private final BroadcasterHandler handler;
public BroadcasterReceiveServlet(BroadcasterHandler handler) {
this.handler = handler;
}
- private static final Pattern PATTERN = Pattern.compile("/(.+)/(.+)/(.+)");
- private static final Pattern PATTERN2 = Pattern.compile("/(.+)/(.+)");
-
@Override
protected void doPut(HttpServletRequest req, HttpServletResponse resp) throws IOException {
handle(req, resp);
@@ -72,7 +67,8 @@ public class BroadcasterReceiveServlet extends HttpServlet {
if (handler != null) {
handler.handle(type, cacheKey, event);
}
- resp.getWriter().write("type:" + type + " name:" + cacheKey + " event:" + event);
+ resp.getWriter().write("Encrypted(type:" + EncryptUtil.encrypt(type) + " name:" + EncryptUtil.encrypt(cacheKey)
+ + " event:" + EncryptUtil.encrypt(event) + ")");
} else if (matcher2.matches()) {
String type = matcher2.group(1);
String event = matcher2.group(2);
@@ -82,10 +78,16 @@ public class BroadcasterReceiveServlet extends HttpServlet {
if (handler != null) {
handler.handle(type, cacheKey, event);
}
- resp.getWriter().write("type:" + type + " name:" + cacheKey + " event:" + event);
+ resp.getWriter().write("Encrypted(type:" + EncryptUtil.encrypt(type) + " name:" + EncryptUtil.encrypt(cacheKey)
+ + " event:" + EncryptUtil.encrypt(event) + ")");
} else {
resp.getWriter().write("not valid uri");
}
resp.getWriter().close();
}
+
+ public interface BroadcasterHandler {
+
+ void handle(String type, String name, String event);
+ }
}