You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/06/05 01:13:04 UTC
[7/7] kylin git commit: KYLIN-1705 code review & cleanup
KYLIN-1705 code review & cleanup
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba7254a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba7254a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba7254a3
Branch: refs/heads/KYLIN-1705-CI
Commit: ba7254a34438e170709a640fa6c5194eb026e7cf
Parents: 5198b87
Author: Yang Li <li...@apache.org>
Authored: Sun Jun 5 09:11:30 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Jun 5 09:11:30 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 2 +
.../common/persistence/ComparableWritable.java | 25 ------
.../kylin/common/persistence/Writable.java | 82 --------------------
.../apache/kylin/common/util/Dictionary.java | 26 ++++++-
core-dictionary/pom.xml | 6 +-
.../apache/kylin/dict/AppendTrieDictionary.java | 36 ++++-----
.../org/apache/kylin/dict/CachedTreeMap.java | 27 ++++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +-
8 files changed, 64 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e300fd7..68e3b6c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -775,6 +775,7 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.dict.append.entry.size", "10000000"));
}
+ // for test
public void setAppendDictEntrySize(int entrySize) {
setProperty("kylin.dict.append.entry.size", String.valueOf(entrySize));
}
@@ -783,6 +784,7 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.dict.append.cache.size", "20"));
}
+ // for test
public void setAppendDictCacheSize(int cacheSize) {
setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-common/src/main/java/org/apache/kylin/common/persistence/ComparableWritable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ComparableWritable.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ComparableWritable.java
deleted file mode 100644
index 5dae9cb..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ComparableWritable.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.persistence;
-
-/**
- * Created by sunyerui on 16/5/11.
- */
-public interface ComparableWritable extends Comparable, Writable {
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java b/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java
deleted file mode 100644
index 7a7fd32..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common.persistence;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Copied from hadoop writable:
- *
- * A serializable object which implements a simple, efficient, serialization
- * protocol, based on {@link DataInput} and {@link DataOutput}.
- *
- * <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
- * framework implements this interface.</p>
- *
- * <p>Implementations typically implement a static <code>read(DataInput)</code>
- * method which constructs a new instance, calls {@link #readFields(DataInput)}
- * and returns the instance.</p>
- *
- * <p>Example:</p>
- * <p><blockquote><pre>
- * public class MyWritable implements Writable {
- * // Some data
- * private int counter;
- * private long timestamp;
- *
- * public void write(DataOutput out) throws IOException {
- * out.writeInt(counter);
- * out.writeLong(timestamp);
- * }
- *
- * public void readFields(DataInput in) throws IOException {
- * counter = in.readInt();
- * timestamp = in.readLong();
- * }
- *
- * public static MyWritable read(DataInput in) throws IOException {
- * MyWritable w = new MyWritable();
- * w.readFields(in);
- * return w;
- * }
- * }
- * </pre></blockquote></p>
- */
-public interface Writable {
- /**
- * Serialize the fields of this object to <code>out</code>.
- *
- * @param out <code>DataOuput</code> to serialize this object into.
- * @throws IOException
- */
- void write(DataOutput out) throws IOException;
-
- /**
- * Deserialize the fields of this object from <code>in</code>.
- *
- * <p>For efficiency, implementations should attempt to re-use storage in the
- * existing object where possible.</p>
- *
- * @param in <code>DataInput</code> to deseriablize this object from.
- * @throws IOException
- */
- void readFields(DataInput in) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index e427555..818b4d8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -18,12 +18,13 @@
package org.apache.kylin.common.util;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
-import org.apache.kylin.common.persistence.Writable;
-
/**
* A bi-way dictionary that maps from dimension/column values to IDs and vice
* versa. By storing IDs instead of real values, the size of cube is
@@ -40,7 +41,7 @@ import org.apache.kylin.common.persistence.Writable;
* @author yangli9
*/
@SuppressWarnings("serial")
-abstract public class Dictionary<T> implements Writable, Serializable {
+abstract public class Dictionary<T> implements Serializable {
// ID with all bit-1 (0xff e.g.) reserved for NULL value
public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
@@ -227,4 +228,23 @@ abstract public class Dictionary<T> implements Writable, Serializable {
}
}
+ /**
+ * Serialize the fields of this object to <code>out</code>.
+ *
+ * @param out <code>DataOuput</code> to serialize this object into.
+ * @throws IOException
+ */
+ public abstract void write(DataOutput out) throws IOException;
+
+ /**
+ * Deserialize the fields of this object from <code>in</code>.
+ *
+ * <p>For efficiency, implementations should attempt to re-use storage in the
+ * existing object where possible.</p>
+ *
+ * @param in <code>DataInput</code> to deseriablize this object from.
+ * @throws IOException
+ */
+ public abstract void readFields(DataInput in) throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 98cd1f8..b0f400e 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -41,13 +41,13 @@
<artifactId>kylin-core-metadata</artifactId>
<version>${project.parent.version}</version>
</dependency>
+
+ <!-- Env & Test -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
+ <scope>provided</scope>
</dependency>
-
- <!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index a51a798..0d5b7df 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -18,21 +18,6 @@
package org.apache.kylin.dict;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ComparableWritable;
-import org.apache.kylin.common.persistence.Writable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
@@ -50,7 +35,21 @@ import java.util.IdentityHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[] to
@@ -69,7 +68,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* @author sunyerui
*/
-@SuppressWarnings({ "rawtypes", "unchecked" })
+@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
public class AppendTrieDictionary<T> extends Dictionary<T> {
public static final byte[] HEAD_MAGIC = new byte[] { 0x41, 0x70, 0x70, 0x65, 0x63, 0x64, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "AppendTrieDict"
@@ -134,7 +133,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
return dictMapBytes;
}
- public static class DictSliceKey implements ComparableWritable {
+ public static class DictSliceKey implements WritableComparable {
byte[] key;
public static DictSliceKey wrap(byte[] key) {
@@ -186,7 +185,6 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
// non-persistent part
transient private int headSize;
- @SuppressWarnings("unused")
transient private int bodyLen;
transient private int sizeChildOffset;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index 81ac82f..43e0950 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -18,28 +18,37 @@
package org.apache.kylin.dict;
-import com.google.common.cache.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.AbstractCollection;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.persistence.ComparableWritable;
-import org.apache.kylin.common.persistence.Writable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ExecutionException;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
/**
* Created by sunyerui on 16/5/2.
* TODO Depends on HDFS for now, ideally just depends on storage interface
*/
-public class CachedTreeMap<K extends ComparableWritable, V extends Writable> extends TreeMap<K, V> implements Writable {
+public class CachedTreeMap<K extends WritableComparable, V extends Writable> extends TreeMap<K, V> implements Writable {
private static final Logger logger = LoggerFactory.getLogger(CachedTreeMap.class);
private final Class<K> keyClazz;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7254a3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index e81df61..0c6a053 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -321,7 +321,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicInteger totalScannedCount = new AtomicInteger(0);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum);
- final boolean compressionResult = KylinConfig.getInstanceFromEnv().getCompressionResult();
+ final boolean compressionResult = cubeSeg.getCubeDesc().getConfig().getCompressionResult();
final CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
builder.setGtScanRequest(scanRequestByteString).setHbaseRawScan(rawScanByteString);
for (IntList intList : hbaseColumnsToGTIntList) {