You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/11 03:18:49 UTC
incubator-kylin git commit: KYLIN-1099 Remove dedup in dictionary
value enumerator
Repository: incubator-kylin
Updated Branches:
refs/heads/1.x-staging 882ca61bc -> a441237af
KYLIN-1099 Remove dedup in dictionary value enumerator
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a441237a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a441237a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a441237a
Branch: refs/heads/1.x-staging
Commit: a441237afbf45736eea74cf6638e34de60a82cea
Parents: 882ca61
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Nov 2 09:55:05 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Nov 11 10:18:13 2015 +0800
----------------------------------------------------------------------
.../dict/ListDictionaryValueEnumerator.java | 50 ++++++++++++++++++++
.../dict/MultipleDictionaryValueEnumerator.java | 10 +---
.../kylin/dict/TableColumnValueEnumerator.java | 13 ++---
.../lookup/ListDictionaryValueEnumerator.java | 50 --------------------
.../job/hadoop/cube/MergeCuboidMapperTest.java | 2 +-
5 files changed, 55 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
new file mode 100644
index 0000000..9ae08a8
--- /dev/null
+++ b/dictionary/src/main/java/org/apache/kylin/dict/ListDictionaryValueEnumerator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.dict.IDictionaryValueEnumerator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * Created by dongli on 10/28/15.
+ */
+public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator {
+ ListIterator<byte[]> listIterator;
+
+ public ListDictionaryValueEnumerator(List<byte[]> list) {
+ listIterator = list.listIterator();
+ }
+
+ @Override
+ public byte[] current() throws IOException {
+ return listIterator.next();
+ }
+
+ @Override
+ public boolean moveNext() throws IOException {
+ return listIterator.hasNext();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index 4cf72d3..13f7394 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -19,12 +19,9 @@
package org.apache.kylin.dict;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
import org.apache.kylin.common.util.Bytes;
import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
/**
@@ -32,7 +29,6 @@ import java.util.List;
*/
@SuppressWarnings("rawtypes")
public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumerator {
- private HashSet<byte[]> dedup = Sets.newHashSet();
private int curDictIndex = 0;
private Dictionary curDict;
private int curKey;
@@ -69,11 +65,7 @@ public class MultipleDictionaryValueEnumerator implements IDictionaryValueEnumer
}
}
- if (dedup.contains(curValue)) {
- return moveNext();
- } else {
- return true;
- }
+ return true;
}
curValue = null;
return false;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
index 64a0e39..42f5791 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
@@ -18,13 +18,11 @@
package org.apache.kylin.dict;
-import com.google.common.collect.Sets;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dict.lookup.ReadableTable;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Set;
/**
* Created by dongli on 10/28/15.
@@ -34,7 +32,6 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator {
private ReadableTable.TableReader reader;
private int colIndex;
private byte[] colValue;
- private Set<String> dedup = Sets.newHashSet();
public TableColumnValueEnumerator(ReadableTable.TableReader reader, int colIndex) {
this.reader = reader;
@@ -55,13 +52,9 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator {
}
colStrValue = split[colIndex];
}
- if (!dedup.contains(colStrValue)) {
- dedup.add(colStrValue);
- colValue = Bytes.toBytes(colStrValue);
- return true;
- } else {
- return moveNext();
- }
+
+ colValue = Bytes.toBytes(colStrValue);
+ return true;
} else {
colValue = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
deleted file mode 100644
index 581ee43..0000000
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/ListDictionaryValueEnumerator.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict.lookup;
-
-import org.apache.kylin.dict.IDictionaryValueEnumerator;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ListIterator;
-
-/**
- * Created by dongli on 10/28/15.
- */
-public class ListDictionaryValueEnumerator implements IDictionaryValueEnumerator {
- ListIterator<byte[]> listIterator;
-
- public ListDictionaryValueEnumerator(List<byte[]> list) {
- listIterator = list.listIterator();
- }
-
- @Override
- public byte[] current() throws IOException {
- return listIterator.next();
- }
-
- @Override
- public boolean moveNext() throws IOException {
- return listIterator.hasNext();
- }
-
- @Override
- public void close() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a441237a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index 7a2ebad..9a1fdfb 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -38,7 +38,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.dict.lookup.ListDictionaryValueEnumerator;
+import org.apache.kylin.dict.ListDictionaryValueEnumerator;
import org.apache.kylin.dict.lookup.ReadableTable.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TblColRef;