You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/11/26 10:39:24 UTC
[kylin] branch kylin-on-druid updated: KYLIN-3694 Bug fix and
revert guava version
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch kylin-on-druid
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-druid by this push:
new d250468 KYLIN-3694 Bug fix and revert guava version
d250468 is described below
commit d2504688a25f1b70e289b01dda16bdd422583ce3
Author: hit-lacus <hi...@126.com>
AuthorDate: Fri Nov 23 10:25:33 2018 +0800
KYLIN-3694 Bug fix and revert guava version
---
assembly/pom.xml | 9 ---------
.../src/main/java/org/apache/kylin/cube/util/CubingUtils.java | 2 +-
.../apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java | 2 +-
.../org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java | 3 ++-
.../engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java | 4 ++--
.../apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java | 4 ++--
.../java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java | 2 +-
.../apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java | 8 ++++----
.../java/org/apache/kylin/engine/spark/SparkFactDistinct.java | 6 +++---
pom.xml | 2 +-
.../apache/kylin/rest/security/KylinAuthenticationProvider.java | 2 +-
.../main/java/org/apache/kylin/storage/druid/DruidSchema.java | 2 +-
.../apache/kylin/storage/druid/read/cursor/HttpDruidClient.java | 4 +++-
13 files changed, 22 insertions(+), 28 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 6dfe165..1e546cb 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -35,11 +35,6 @@
<dependencies>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-source-hive</artifactId>
</dependency>
@@ -228,10 +223,6 @@
<pattern>org.apache.commons.jocl</pattern>
<shadedPattern>${shadeBase}.org.apache.commons.jocl</shadedPattern>
</relocation>
- <relocation>
- <pattern>com.google.guava</pattern>
- <shadedPattern>${shadeBase}.com.google.guava</shadedPattern>
- </relocation>
</relocations>
<filters>
<filter>
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 8f4a1f6..f78e92b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -87,7 +87,7 @@ public class CubingUtils {
Hasher hc = hf.newHasher();
final String cell = row.get(flatDesc.getRowKeyColumnIndexes()[i]);
if (cell != null) {
- row_hashcodes[i] = hc.putUnencodedChars(cell).hash().asBytes();
+ row_hashcodes[i] = hc.putString(cell).hash().asBytes();
} else {
row_hashcodes[i] = hc.putInt(0).hash().asBytes();
}
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index b9fb984..c3642cb 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -147,7 +147,7 @@ public class CuboidStatsUtilTest {
@Test
public void createDirectChildrenCacheStressTest() {
- Stopwatch sw = Stopwatch.createStarted();
+ Stopwatch sw = new Stopwatch();
Set<Long> cuboidSet = generateMassCuboidSet();
System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS));
sw.reset();
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 81fe71f..0e15687 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -167,7 +167,8 @@ public class AggregationCacheMemSizeTest {
}
}
- @Test
+ // Ignore because of OOM
+ // @Test
public void testEstimateMemSize() throws InterruptedException {
int scale = Integer.parseInt(System.getProperty("scale", "1"));
scale = Math.max(1, Math.min(10, scale));
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
index 0814281..1b32944 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
@@ -146,7 +146,7 @@ public class CalculateStatsFromBaseCuboidMapper extends KylinMapper<Text, Text,
Hasher hc = hf.newHasher();
String colValue = row[i];
if (colValue != null) {
- rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes();
+ rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
} else {
rowHashCodes[i] = hc.putInt(0).hash().asBytes();
}
@@ -170,7 +170,7 @@ public class CalculateStatsFromBaseCuboidMapper extends KylinMapper<Text, Text,
String colValue = row[i];
if (colValue == null)
colValue = "0";
- byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a)
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 6c5edda..7bffce7 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -350,7 +350,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
Hasher hc = hf.newHasher();
String colValue = row[rowkeyColIndex[i]];
if (colValue != null) {
- rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes();
+ rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
} else {
rowHashCodes[i] = hc.putInt(0).hash().asBytes();
}
@@ -374,7 +374,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
String colValue = row[rowkeyColIndex[i]];
if (colValue == null)
colValue = "0";
- byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a)
}
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index fcf50c7..916f9fa 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -105,7 +105,7 @@ public class CubeSamplingTest {
int x = 0;
for (String field : row) {
Hasher hc = hf.newHasher();
- row_index[x++] = hc.putUnencodedChars(field).hash().asBytes();
+ row_index[x++] = hc.putString(field).hash().asBytes();
}
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
index a82f73d..97e356d 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
@@ -113,7 +113,7 @@ public class NewCubeSamplingMethodTest {
int x = 0;
for (String field : row) {
Hasher hc = hf.newHasher();
- colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes();
+ colHashValues[x++] = hc.putString(field).hash().asBytes();
}
Hasher hc = hf.newHasher();
@@ -139,7 +139,7 @@ public class NewCubeSamplingMethodTest {
int x = 0;
for (String field : row) {
Hasher hc = hf2.newHasher();
- byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes();
+ byte[] bytes = hc.putString(x + field).hash().asBytes();
valueHashLong[x++] = Bytes.toLong(bytes);
}
@@ -213,7 +213,7 @@ public class NewCubeSamplingMethodTest {
int x = 0;
for (String field : row) {
Hasher hc = hashFunction.newHasher();
- colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes();
+ colHashValues[x++] = hc.putString(field).hash().asBytes();
}
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
@@ -230,7 +230,7 @@ public class NewCubeSamplingMethodTest {
int x = 0;
for (String field : row) {
Hasher hc = hashFunction.newHasher();
- byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes();
+ byte[] bytes = hc.putString(x + field).hash().asBytes();
hashValuesLong[x++] = Bytes.toLong(bytes);
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index 878ae7f..31cadaa 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -528,7 +528,7 @@ public class SparkFactDistinct extends AbstractApplication implements Serializab
Hasher hc = hf.newHasher();
String colValue = row[rowkeyColIndex[i]];
if (colValue != null) {
- rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes();
+ rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
} else {
rowHashCodes[i] = hc.putInt(0).hash().asBytes();
}
@@ -551,8 +551,8 @@ public class SparkFactDistinct extends AbstractApplication implements Serializab
Hasher hc = hf.newHasher();
String colValue = row[rowkeyColIndex[i]];
if (colValue == null)
- colValue = "0";
- byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+ colValue = "0";//
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a)
}
diff --git a/pom.xml b/pom.xml
index 7b91a5d..d957f07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,7 +82,7 @@
<zookeeper.version>3.4.13</zookeeper.version>
<curator.version>2.12.0</curator.version>
<jsr305.version>3.0.1</jsr305.version>
- <guava.version>16.0.1</guava.version>
+ <guava.version>14.0</guava.version>
<jsch.version>0.1.54</jsch.version>
<commons-cli.version>1.2</commons-cli.version>
<commons-lang.version>2.6</commons-lang.version>
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index 92e25c8..dd9cbad 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -78,7 +78,7 @@ public class KylinAuthenticationProvider implements AuthenticationProvider {
@Override
public Authentication authenticate(Authentication authentication) throws AuthenticationException {
- byte[] hashKey = hf.hashUnencodedChars(authentication.getName() + authentication.getCredentials()).asBytes();
+ byte[] hashKey = hf.hashString(authentication.getName() + authentication.getCredentials()).asBytes();
String userKey = Arrays.toString(hashKey);
if (userService.isEvictCacheFlag()) {
diff --git a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
index 54c8591..8f473c4 100644
--- a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
+++ b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
@@ -99,7 +99,7 @@ public class DruidSchema implements NameMapping {
if (!segment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) {
return Intervals.utc(0, segment.getCreateTimeUTC());
}
- return Intervals.utc(segment.getTSRange().start.v, segment.getTSRange().start.v);
+ return Intervals.utc(segment.getTSRange().start.v, segment.getTSRange().end.v);
}
public static String getDataSource(CubeDesc cubeDesc) {
diff --git a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
index 9af7c63..6bf9d5d 100644
--- a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
+++ b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.common.base.Throwables;
import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
import com.google.common.util.concurrent.ListenableFuture;
import io.druid.java.util.common.IAE;
import io.druid.java.util.common.RE;
@@ -162,7 +163,8 @@ public class HttpDruidClient<T> implements DruidClient<T> {
try {
// An empty byte array is put at the end to give the SequenceInputStream.close() as something to close out
// after done is set to true, regardless of the rest of the stream's state.
- queue.put(ByteSource.empty().openStream());
+ ByteSource empty = ByteStreams.asByteSource(new byte[]{});
+ queue.put(empty.openStream());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);