You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/13 03:49:24 UTC
kylin git commit: KYLIN-2428 clean guava
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2428 ae3dff85c -> e911ef473
KYLIN-2428 clean guava
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e911ef47
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e911ef47
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e911ef47
Branch: refs/heads/KYLIN-2428
Commit: e911ef4730b6fcdf7405216b30500a5e0e2ff570
Parents: ae3dff8
Author: Billy Liu <bi...@apache.org>
Authored: Mon Feb 13 11:49:19 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Mon Feb 13 11:49:19 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/spark/KylinKryoRegistrator.java | 10 +++-------
.../kylin/engine/spark/SparkCubingByLayer.java | 6 +++---
.../apache/kylin/engine/spark/SparkExecutable.java | 15 ++++++++-------
.../engine/spark/cube/ListBackedCuboidWriter.java | 4 +---
.../kylin/engine/spark/util/IteratorUtils.java | 4 +---
5 files changed, 16 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 322e835..f5f4f36 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -19,15 +19,11 @@
package org.apache.kylin.engine.spark;
import com.esotericsoftware.kryo.Kryo;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-//import org.apache.kylin.measure.MeasureIngester;
import org.apache.spark.serializer.KryoRegistrator;
-//import org.reflections.Reflections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-//import java.io.Serializable;
+import java.util.LinkedHashSet;
import java.util.Set;
/**
@@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
@Override
public void registerClasses(Kryo kryo) {
- Set<Class> kyroClasses = Sets.newLinkedHashSet();
+ Set<Class> kyroClasses = new LinkedHashSet();
kyroClasses.add(byte[].class);
kyroClasses.add(int[].class);
kyroClasses.add(byte[][].class);
@@ -74,7 +70,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
- kyroClasses.add(Hashing.murmur3_128().getClass());
+// kyroClasses.add(Hashing.murmur3_128().getClass());
kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
kyroClasses.add(scala.math.BigDecimal.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 317d2bd..9329c08 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -17,7 +17,6 @@
*/
package org.apache.kylin.engine.spark;
-import com.google.common.collect.Lists;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -72,6 +71,7 @@ import java.io.File;
import java.io.FileFilter;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -333,7 +333,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
}
- private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = Lists.newArrayListWithCapacity(0);
+ private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = new ArrayList(0);
class CuboidFlatMap implements PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> {
@@ -370,7 +370,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
return EMTPY_ITERATOR;
}
- List<Tuple2<ByteArray, Object[]>> tuples = Lists.newArrayListWithCapacity(myChildren.size());
+ List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size());
for (Long child : myChildren) {
Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 4ab846f..015b480 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -20,8 +20,7 @@ package org.apache.kylin.engine.spark;
import java.io.File;
import java.util.Map;
-import jodd.util.StringUtil;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Logger;
@@ -31,8 +30,6 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-
/**
*/
public class SparkExecutable extends AbstractExecutable {
@@ -74,12 +71,16 @@ public class SparkExecutable extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final KylinConfig config = context.getConfig();
- Preconditions.checkNotNull(config.getSparkHome());
- Preconditions.checkNotNull(config.getKylinJobJarPath());
+ if (config.getSparkHome() == null) {
+ throw new NullPointerException();
+ }
+ if (config.getKylinJobJarPath() == null) {
+ throw new NullPointerException();
+ }
String jars = this.getParam(JARS);
String hadoopConf = "/etc/hadoop/conf";
- if (StringUtil.isNotEmpty(config.getHadoopConfDir())) {
+ if (StringUtils.isNotEmpty(config.getHadoopConfDir())) {
hadoopConf = config.getHadoopConfDir();
} else {
String hiveConf = ClassLoader.getSystemClassLoader().getResource("hive-site.xml").getFile().toString();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
index 531670f..a2740bf 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java
@@ -23,8 +23,6 @@ import java.util.ArrayList;
import org.apache.kylin.engine.spark.SparkCuboidWriter;
import org.apache.kylin.gridtable.GTRecord;
-import com.google.common.collect.Lists;
-
import scala.Tuple2;
/**
@@ -35,7 +33,7 @@ public class ListBackedCuboidWriter implements SparkCuboidWriter {
private final TupleConverter tupleConverter;
public ListBackedCuboidWriter(TupleConverter tupleConverter) {
- this.result = Lists.newArrayList();
+ this.result = new ArrayList();
this.tupleConverter = tupleConverter;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e911ef47/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
index 13464a0..a8a4d28 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java
@@ -24,8 +24,6 @@ import java.util.NoSuchElementException;
import org.apache.spark.api.java.function.Function;
-import com.google.common.collect.Lists;
-
import scala.Tuple2;
/**
@@ -47,7 +45,7 @@ public class IteratorUtils {
if (!hasNext()) {
throw new NoSuchElementException();
}
- final LinkedList<V> values = Lists.newLinkedList();
+ final LinkedList<V> values = new LinkedList();
K currentKey = current._1();
values.add(current._2());
while (input.hasNext()) {