You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/22 15:35:26 UTC
incubator-kylin git commit: KYLIN-875 Refactor core-dictionary,
drop hadoop/hbase dependency
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 ecd78c5dd -> 5eda06529
KYLIN-875 Refactor core-dictionary, drop hadoop/hbase dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5eda0652
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5eda0652
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5eda0652
Branch: refs/heads/0.8
Commit: 5eda06529429aa396b233ee385f05a0896401cc0
Parents: ecd78c5
Author: Yang Li <li...@apache.org>
Authored: Wed Jul 22 21:35:03 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Jul 22 21:35:03 2015 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/Writable.java | 82 ++++++++++++++++++++
core-dictionary/pom.xml | 50 +-----------
.../apache/kylin/dict/DateStrDictionary.java | 2 +-
.../java/org/apache/kylin/dict/Dictionary.java | 8 +-
.../apache/kylin/dict/DictionarySerializer.java | 22 ++++--
.../apache/kylin/dict/TimeStrDictionary.java | 2 +-
.../apache/kylin/dict/lookup/SnapshotTable.java | 6 +-
.../invertedindex/model/IIKeyValueCodec.java | 7 +-
8 files changed, 111 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java b/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java
new file mode 100644
index 0000000..3f57986
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/Writable.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import java.io.DataOutput;
+import java.io.DataInput;
+import java.io.IOException;
+
+/**
+ * Copied from hadoop writable:
+ *
+ * A serializable object which implements a simple, efficient, serialization
+ * protocol, based on {@link DataInput} and {@link DataOutput}.
+ *
+ * <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
+ * framework implements this interface.</p>
+ *
+ * <p>Implementations typically implement a static <code>read(DataInput)</code>
+ * method which constructs a new instance, calls {@link #readFields(DataInput)}
+ * and returns the instance.</p>
+ *
+ * <p>Example:</p>
+ * <p><blockquote><pre>
+ * public class MyWritable implements Writable {
+ * // Some data
+ * private int counter;
+ * private long timestamp;
+ *
+ * public void write(DataOutput out) throws IOException {
+ * out.writeInt(counter);
+ * out.writeLong(timestamp);
+ * }
+ *
+ * public void readFields(DataInput in) throws IOException {
+ * counter = in.readInt();
+ * timestamp = in.readLong();
+ * }
+ *
+ * public static MyWritable read(DataInput in) throws IOException {
+ * MyWritable w = new MyWritable();
+ * w.readFields(in);
+ * return w;
+ * }
+ * }
+ * </pre></blockquote></p>
+ */
+public interface Writable {
+ /**
+ * Serialize the fields of this object to <code>out</code>.
+ *
+ * @param out <code>DataOuput</code> to serialize this object into.
+ * @throws IOException
+ */
+ void write(DataOutput out) throws IOException;
+
+ /**
+ * Deserialize the fields of this object from <code>in</code>.
+ *
+ * <p>For efficiency, implementations should attempt to re-use storage in the
+ * existing object where possible.</p>
+ *
+ * @param in <code>DataInput</code> to deseriablize this object from.
+ * @throws IOException
+ */
+ void readFields(DataInput in) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 649f58b..4ea56b9 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -50,59 +50,11 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- <!-- protobuf version conflict with hbase-->
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <version>${hive-hcatalog.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${hbase-hadoop2.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet.jsp</groupId>
- <artifactId>jsp-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 0ee1d06..6ff7cd8 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -174,7 +174,7 @@ public class DateStrDictionary extends Dictionary<String> {
}
@Override
- public boolean containedBy(Dictionary other) {
+ public boolean containedBy(Dictionary<?> other) {
return this.equals(other);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index 1df950d..45cd361 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -18,12 +18,12 @@
package org.apache.kylin.dict;
-import org.apache.hadoop.io.Writable;
-import org.apache.kylin.common.util.BytesUtil;
-
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import org.apache.kylin.common.persistence.Writable;
+import org.apache.kylin.common.util.BytesUtil;
+
/**
* A bi-way dictionary that maps from dimension/column values to IDs and vice
* versa. By storing IDs instead of real values, the size of cube is
@@ -67,7 +67,7 @@ abstract public class Dictionary<T> implements Writable {
/**
* @return true if each entry of this dict is contained by the dict in param
*/
- abstract public boolean containedBy(Dictionary another);
+ abstract public boolean containedBy(Dictionary<?> another);
/**
* Convenient form of <code>getIdFromValue(value, 0)</code>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
index a8ad2fa..42c1336 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
@@ -1,10 +1,16 @@
package org.apache.kylin.dict;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ClassUtil;
-import java.io.*;
-
/**
*/
public final class DictionarySerializer {
@@ -15,7 +21,7 @@ public final class DictionarySerializer {
try {
final DataInputStream dataInputStream = new DataInputStream(inputStream);
final String type = dataInputStream.readUTF();
- final Dictionary dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
+ final Dictionary<?> dictionary = ClassUtil.forName(type, Dictionary.class).newInstance();
dictionary.readFields(dataInputStream);
return dictionary;
} catch (Exception e) {
@@ -23,8 +29,8 @@ public final class DictionarySerializer {
}
}
- public static Dictionary<?> deserialize(ImmutableBytesWritable dictBytes) {
- return deserialize(new ByteArrayInputStream(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
+ public static Dictionary<?> deserialize(ByteArray dictBytes) {
+ return deserialize(new ByteArrayInputStream(dictBytes.array(), dictBytes.offset(), dictBytes.length()));
}
public static void serialize(Dictionary<?> dict, OutputStream outputStream) {
@@ -38,13 +44,13 @@ public final class DictionarySerializer {
}
}
- public static ImmutableBytesWritable serialize(Dictionary<?> dict) {
+ public static ByteArray serialize(Dictionary<?> dict) {
try {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(baos);
out.writeUTF(dict.getClass().getName());
dict.write(out);
- return new ImmutableBytesWritable(baos.toByteArray());
+ return new ByteArray(baos.toByteArray());
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index 92f3a3c..6b58354 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -106,7 +106,7 @@ public class TimeStrDictionary extends Dictionary<String> {
}
@Override
- public boolean containedBy(Dictionary other) {
+ public boolean containedBy(Dictionary<?> other) {
return this.equals(other);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index e2205b9..de2336e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.dict.Dictionary;
@@ -35,6 +34,7 @@ import org.apache.kylin.source.ReadableTable;
import java.io.DataInput;
import java.io.DataOutput;
+import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
@@ -100,11 +100,11 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
}
public String getResourcePath() {
- return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new Path(signature.getPath()).getName() + "/" + uuid + ".snapshot";
+ return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName() + "/" + uuid + ".snapshot";
}
public String getResourceDir() {
- return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new Path(signature.getPath()).getName();
+ return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5eda0652/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index b236879..ae23558 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -20,8 +20,10 @@ package org.apache.kylin.invertedindex.model;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Array;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionarySerializer;
@@ -72,7 +74,8 @@ public class IIKeyValueCodec implements KeyValueCodec {
if (dictionary == null) {
return new IIRow(key, value, new ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY));
} else {
- return new IIRow(key, value, DictionarySerializer.serialize(dictionary));
+ ByteArray bytes = DictionarySerializer.serialize(dictionary);
+ return new IIRow(key, value, new ImmutableBytesWritable(bytes.array(), bytes.offset(), bytes.length()));
}
}
@@ -196,7 +199,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
} else {
final ImmutableBytesWritable dictBytes = row.getDictionary();
if (dictBytes.getLength() != 0) {
- final Dictionary<?> dictionary = DictionarySerializer.deserialize(dictBytes);
+ final Dictionary<?> dictionary = DictionarySerializer.deserialize(new ByteArray(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), dictionary.getMaxId() - dictionary.getMinId() + 1, 0);
c.fromBytes(row.getValue());
valueContainers[curCol] = c;