You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/14 01:01:07 UTC

[32/50] [abbrv] incubator-kylin git commit: refactor

refactor


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab458df4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab458df4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab458df4

Branch: refs/heads/streaming-localdict
Commit: ab458df401c23ea86dd5537e35c6d93669961246
Parents: 7b48b51
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Mar 12 17:47:02 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Mar 12 17:47:02 2015 +0800

----------------------------------------------------------------------
 .../invertedindex/index/BitMapContainer.java    | 382 +++++++++----------
 .../kylin/invertedindex/index/SliceBuilder.java |   3 -
 .../invertedindex/index/TableRecordInfo.java    |  22 +-
 .../invertedindex/model/IIKeyValueCodec.java    | 164 +++-----
 .../apache/kylin/invertedindex/model/IIRow.java |  67 ++++
 .../invertedindex/model/KeyValueCodec.java      |  49 +++
 .../kylin/invertedindex/model/KeyValuePair.java |  78 ----
 .../apache/kylin/invertedindex/tools/IICLI.java |  16 +-
 .../invertedindex/InvertedIndexLocalTest.java   |  40 +-
 .../invertedindex/InvertedIndexReducer.java     |   6 +-
 .../storage/hbase/HBaseClientKVIterator.java    |  13 +-
 .../endpoint/HbaseServerKVIterator.java         | 110 +++---
 .../hbase/coprocessor/endpoint/IIEndpoint.java  |  24 +-
 .../invertedindex/IIStreamBuilder.java          |  57 +--
 14 files changed, 489 insertions(+), 542 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
index 066a3c8..ad3b0c1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
@@ -34,194 +34,194 @@ import org.apache.kylin.dict.Dictionary;
 /**
  * @author yangli9
  */
-public class BitMapContainer implements ColumnValueContainer {
-
-    int valueLen;
-    int nValues;
-    int size;
-    ConciseSet[] sets;
-    boolean closedForChange;
-
-    transient byte[] temp;
-
-    public BitMapContainer(TableRecordInfoDigest digest, int col) {
-        this.valueLen = digest.length(col);
-        this.size = 0;
-        this.nValues = digest.getMaxID(col) + 1;
-        this.sets = null;
-        this.closedForChange = false;
-
-        this.temp = new byte[valueLen];
-    }
-
-    @Override
-    public void append(ImmutableBytesWritable valueBytes) {
-        int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
-        append(value);
-    }
-
-    public void append(int value) {
-        checkUpdateMode();
-        if (value == Dictionary.NULL_ID[valueLen]) {
-            value = nValues; // set[nValues] holds NULL
-        }
-        sets[value].add(size);
-        size++;
-    }
-
-    @Override
-    public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
-        int value = getValueIntAt(i);
-        BytesUtil.writeUnsigned(value, temp, 0, valueLen);
-        valueBytes.set(temp, 0, valueLen);
-    }
-
-    @Override
-    public ConciseSet getBitMap(Integer startId, Integer endId) {
-        if (startId == null && endId == null) {
-            return sets[this.nValues];
-        }
-
-        int start = 0;
-        int end = this.nValues - 1;
-        if (startId != null) {
-            start = startId;
-        }
-        if (endId != null) {
-            end = endId;
-        }
-
-        ConciseSet ret = new ConciseSet();
-        for (int i = start; i <= end; ++i) {
-            ConciseSet temp = getBitMap(i);
-            ret.addAll(temp);
-        }
-        return ret;
-    }
-
-    private ConciseSet getBitMap(int valueId) {
-        if (valueId >= 0 && valueId <= getMaxValueId())
-            return sets[valueId];
-        else
-            return sets[this.nValues];
-    }
-
-    @Override
-    public int getMaxValueId() {
-        return this.nValues - 1;
-    }
-
-    public int getValueIntAt(int i) {
-        for (int v = 0; v < nValues; v++) {
-            if (sets[v].contains(i)) {
-                return v;
-            }
-        }
-        // if v is not in [0..nValues-1], then it must be nValue (NULL)
-        return Dictionary.NULL_ID[valueLen];
-    }
-
-    private void checkUpdateMode() {
-        if (isClosedForChange()) {
-            throw new IllegalStateException();
-        }
-        if (sets == null) {
-            sets = new ConciseSet[nValues + 1];
-            for (int i = 0; i <= nValues; i++) {
-                sets[i] = new ConciseSet();
-            }
-        }
-    }
-
-    private boolean isClosedForChange() {
-        return closedForChange;
-    }
-
-    @Override
-    public void closeForChange() {
-        closedForChange = true;
-    }
-
-    @Override
-    public int getSize() {
-        return size;
-    }
-
-    public List<ImmutableBytesWritable> toBytes() {
-        if (isClosedForChange() == false)
-            closeForChange();
-
-        List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
-        for (int i = 0; i <= nValues; i++) {
-            r.add(setToBytes(sets[i]));
-        }
-        return r;
-    }
-
-    public void fromBytes(List<ImmutableBytesWritable> bytes) {
-        assert nValues + 1 == bytes.size();
-        sets = new ConciseSet[nValues + 1];
-        size = 0;
-        for (int i = 0; i <= nValues; i++) {
-            sets[i] = bytesToSet(bytes.get(i));
-            size += sets[i].size();
-        }
-        closedForChange = true;
-    }
-
-    private ImmutableBytesWritable setToBytes(ConciseSet set) {
-        byte[] array;
-        if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
-                           // set is empty
-            array = BytesUtil.EMPTY_BYTE_ARRAY;
-        else
-            array = set.toByteBuffer().array();
-        return new ImmutableBytesWritable(array);
-    }
-
-    private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
-        if (bytes.get() == null || bytes.getLength() == 0) {
-            return new ConciseSet();
-        } else {
-            IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
-            int[] words = new int[intBuffer.capacity()];
-            intBuffer.get(words);
-            return new ConciseSet(words, false);
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + (closedForChange ? 1231 : 1237);
-        result = prime * result + nValues;
-        result = prime * result + Arrays.hashCode(sets);
-        result = prime * result + size;
-        result = prime * result + valueLen;
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj)
-            return true;
-        if (obj == null)
-            return false;
-        if (getClass() != obj.getClass())
-            return false;
-        BitMapContainer other = (BitMapContainer) obj;
-        if (closedForChange != other.closedForChange)
-            return false;
-        if (nValues != other.nValues)
-            return false;
-        if (!Arrays.equals(sets, other.sets))
-            return false;
-        if (size != other.size)
-            return false;
-        if (valueLen != other.valueLen)
-            return false;
-        return true;
-    }
-
-}
+//public class BitMapContainer implements ColumnValueContainer {
+//
+//    int valueLen;
+//    int nValues;
+//    int size;
+//    ConciseSet[] sets;
+//    boolean closedForChange;
+//
+//    transient byte[] temp;
+//
+//    public BitMapContainer(TableRecordInfoDigest digest, int col) {
+//        this.valueLen = digest.length(col);
+//        this.size = 0;
+//        this.nValues = digest.getMaxID(col) + 1;
+//        this.sets = null;
+//        this.closedForChange = false;
+//
+//        this.temp = new byte[valueLen];
+//    }
+//
+//    @Override
+//    public void append(ImmutableBytesWritable valueBytes) {
+//        int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
+//        append(value);
+//    }
+//
+//    public void append(int value) {
+//        checkUpdateMode();
+//        if (value == Dictionary.NULL_ID[valueLen]) {
+//            value = nValues; // set[nValues] holds NULL
+//        }
+//        sets[value].add(size);
+//        size++;
+//    }
+//
+//    @Override
+//    public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
+//        int value = getValueIntAt(i);
+//        BytesUtil.writeUnsigned(value, temp, 0, valueLen);
+//        valueBytes.set(temp, 0, valueLen);
+//    }
+//
+//    @Override
+//    public ConciseSet getBitMap(Integer startId, Integer endId) {
+//        if (startId == null && endId == null) {
+//            return sets[this.nValues];
+//        }
+//
+//        int start = 0;
+//        int end = this.nValues - 1;
+//        if (startId != null) {
+//            start = startId;
+//        }
+//        if (endId != null) {
+//            end = endId;
+//        }
+//
+//        ConciseSet ret = new ConciseSet();
+//        for (int i = start; i <= end; ++i) {
+//            ConciseSet temp = getBitMap(i);
+//            ret.addAll(temp);
+//        }
+//        return ret;
+//    }
+//
+//    private ConciseSet getBitMap(int valueId) {
+//        if (valueId >= 0 && valueId <= getMaxValueId())
+//            return sets[valueId];
+//        else
+//            return sets[this.nValues];
+//    }
+//
+//    @Override
+//    public int getMaxValueId() {
+//        return this.nValues - 1;
+//    }
+//
+//    public int getValueIntAt(int i) {
+//        for (int v = 0; v < nValues; v++) {
+//            if (sets[v].contains(i)) {
+//                return v;
+//            }
+//        }
+//        // if v is not in [0..nValues-1], then it must be nValue (NULL)
+//        return Dictionary.NULL_ID[valueLen];
+//    }
+//
+//    private void checkUpdateMode() {
+//        if (isClosedForChange()) {
+//            throw new IllegalStateException();
+//        }
+//        if (sets == null) {
+//            sets = new ConciseSet[nValues + 1];
+//            for (int i = 0; i <= nValues; i++) {
+//                sets[i] = new ConciseSet();
+//            }
+//        }
+//    }
+//
+//    private boolean isClosedForChange() {
+//        return closedForChange;
+//    }
+//
+//    @Override
+//    public void closeForChange() {
+//        closedForChange = true;
+//    }
+//
+//    @Override
+//    public int getSize() {
+//        return size;
+//    }
+//
+//    public List<ImmutableBytesWritable> toBytes() {
+//        if (isClosedForChange() == false)
+//            closeForChange();
+//
+//        List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
+//        for (int i = 0; i <= nValues; i++) {
+//            r.add(setToBytes(sets[i]));
+//        }
+//        return r;
+//    }
+//
+//    public void fromBytes(List<ImmutableBytesWritable> bytes) {
+//        assert nValues + 1 == bytes.size();
+//        sets = new ConciseSet[nValues + 1];
+//        size = 0;
+//        for (int i = 0; i <= nValues; i++) {
+//            sets[i] = bytesToSet(bytes.get(i));
+//            size += sets[i].size();
+//        }
+//        closedForChange = true;
+//    }
+//
+//    private ImmutableBytesWritable setToBytes(ConciseSet set) {
+//        byte[] array;
+//        if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
+//                           // set is empty
+//            array = BytesUtil.EMPTY_BYTE_ARRAY;
+//        else
+//            array = set.toByteBuffer().array();
+//        return new ImmutableBytesWritable(array);
+//    }
+//
+//    private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
+//        if (bytes.get() == null || bytes.getLength() == 0) {
+//            return new ConciseSet();
+//        } else {
+//            IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
+//            int[] words = new int[intBuffer.capacity()];
+//            intBuffer.get(words);
+//            return new ConciseSet(words, false);
+//        }
+//    }
+//
+//    @Override
+//    public int hashCode() {
+//        final int prime = 31;
+//        int result = 1;
+//        result = prime * result + (closedForChange ? 1231 : 1237);
+//        result = prime * result + nValues;
+//        result = prime * result + Arrays.hashCode(sets);
+//        result = prime * result + size;
+//        result = prime * result + valueLen;
+//        return result;
+//    }
+//
+//    @Override
+//    public boolean equals(Object obj) {
+//        if (this == obj)
+//            return true;
+//        if (obj == null)
+//            return false;
+//        if (getClass() != obj.getClass())
+//            return false;
+//        BitMapContainer other = (BitMapContainer) obj;
+//        if (closedForChange != other.closedForChange)
+//            return false;
+//        if (nValues != other.nValues)
+//            return false;
+//        if (!Arrays.equals(sets, other.sets))
+//            return false;
+//        if (size != other.size)
+//            return false;
+//        if (valueLen != other.valueLen)
+//            return false;
+//        return true;
+//    }
+//
+//}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index e217df0..72050a1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -61,9 +61,6 @@ public class SliceBuilder {
 		// reset for next slice
 		nRecords = 0;
 		containers = new ColumnValueContainer[nColumns];
-		for (int i : info.getDescriptor().getBitmapColumns()) {
-			containers[i] = new BitMapContainer(info.getDigest(), i);
-		}
 		for (int i : info.getDescriptor().getValueColumns()) {
 			containers[i] = new CompressedValueContainer(info.getDigest(), i,
 					nRecordsCap);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 0cfb05e..3e72f43 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -43,7 +43,7 @@ public class TableRecordInfo {
     final IIDesc desc;
 
     final TableRecordInfoDigest digest;
-    final Map<TblColRef, Dictionary<?>> dictionaryMap;
+    final Map<Integer, Dictionary<?>> dictionaryMap;
 
     public TableRecordInfo(IISegment iiSegment) {
 
@@ -59,10 +59,12 @@ public class TableRecordInfo {
                 measureCodecMap.put(tblColRef, FixedLenMeasureCodec.get(col.getType()));
             } else {
                 String dictPath = iiSegment.getDictResPath(tblColRef);
-                try {
-                    dictionaryMap.put(tblColRef, dictMgr.getDictionary(dictPath));
-                } catch (IOException e) {
-                    throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
+                if (dictPath != null) {
+                    try {
+                        dictionaryMap.put(index, dictMgr.getDictionary(dictPath));
+                    } catch (IOException e) {
+                        throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
+                    }
                 }
             }
             index++;
@@ -71,7 +73,7 @@ public class TableRecordInfo {
         digest = createDigest(dictionaryMap, measureCodecMap);
     }
 
-    public TableRecordInfo(IIDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+    public TableRecordInfo(IIDesc desc, Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
         this.desc = desc;
         this.dictionaryMap = dictionaryMap;
         this.digest = createDigest(dictionaryMap, measureCodecMap);
@@ -81,7 +83,7 @@ public class TableRecordInfo {
         return digest;
     }
 
-    private TableRecordInfoDigest createDigest(Map<TblColRef, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
+    private TableRecordInfoDigest createDigest(Map<Integer, Dictionary<?>> dictionaryMap, Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap) {
         int nColumns = getColumns().size();
         boolean[] isMetric = new boolean[nColumns];
         int[] lengths = new int[nColumns];
@@ -97,10 +99,10 @@ public class TableRecordInfo {
                     lengths[i] = fixedLenMeasureCodec.getLength();
                 }
             } else {
-                final Dictionary<?> dictionary = dictionaryMap.get(tblColRef);
+                final Dictionary<?> dictionary = dictionaryMap.get(i);
                 if (dictionary != null) {
                     lengths[i] = dictionary.getSizeOfId();
-                    dictMaxIds[i] = dictionaryMap.get(tblColRef).getMaxId();
+                    dictMaxIds[i] = dictionary.getMaxId();
                 }
             }
         }
@@ -149,7 +151,7 @@ public class TableRecordInfo {
     @SuppressWarnings("unchecked")
     public Dictionary<String> dict(int col) {
         // yes, all dictionaries are string based
-        return (Dictionary<String>) dictionaryMap.get(desc.listAllColumns().get(col));
+        return (Dictionary<String>) dictionaryMap.get(col);
     }
 
     public int getTimestampColumn() {

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 8ea4e70..2ebd53d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -18,41 +18,37 @@
 
 package org.apache.kylin.invertedindex.model;
 
-import java.util.*;
-
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-
-import com.google.common.collect.Lists;
-
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.invertedindex.index.ColumnValueContainer;
+import org.apache.kylin.invertedindex.index.CompressedValueContainer;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+
+import java.util.*;
 
 /**
  * @author yangli9
  */
-public class IIKeyValueCodec {
+public class IIKeyValueCodec implements KeyValueCodec {
 
 	public static final int SHARD_LEN = 2;
 	public static final int TIMEPART_LEN = 8;
 	public static final int COLNO_LEN = 2;
 
-	private TableRecordInfoDigest infoDigest;
-
-	public IIKeyValueCodec(TableRecordInfoDigest digest) {
-		this.infoDigest = digest;
+	public IIKeyValueCodec() {
 	}
 
-	public Collection<KeyValuePair> encodeKeyValue(Slice slice) {
-		ArrayList<KeyValuePair> result = Lists
+    @Override
+	public Collection<IIRow> encodeKeyValue(Slice slice) {
+		ArrayList<IIRow> result = Lists
 				.newArrayList();
 		ColumnValueContainer[] containers = slice.getColumnValueContainers();
 		for (int col = 0; col < containers.length; col++) {
-			if (containers[col] instanceof BitMapContainer) {
-				result.addAll(collectKeyValues(slice, col, (BitMapContainer) containers[col]));
-			} else if (containers[col] instanceof CompressedValueContainer) {
+			if (containers[col] instanceof CompressedValueContainer) {
 				result.add(collectKeyValues(slice, col, (CompressedValueContainer) containers[col]));
 			} else {
 				throw new IllegalArgumentException("Unknown container class "
@@ -62,37 +58,21 @@ public class IIKeyValueCodec {
 		return result;
 	}
 
-	private KeyValuePair collectKeyValues(Slice slice,
+	private IIRow collectKeyValues(Slice slice,
 			int col,
 			CompressedValueContainer container) {
-		ImmutableBytesWritable key = encodeKey(slice.getShard(),
-				slice.getTimestamp(), col, -1);
+		ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col);
 		ImmutableBytesWritable value = container.toBytes();
-        return new KeyValuePair(col, key, value);
-	}
-
-	private List<KeyValuePair> collectKeyValues(Slice slice,
-			int col,
-			BitMapContainer container) {
-		List<ImmutableBytesWritable> values = container.toBytes();
-        ArrayList<KeyValuePair> list = Lists.newArrayListWithExpectedSize(values.size());
-		for (int v = 0; v < values.size(); v++) {
-			ImmutableBytesWritable key = encodeKey(slice.getShard(),
-					slice.getTimestamp(), col, v);
-            list.add(new KeyValuePair(col, key, values.get(v)));
-		}
-        return list;
+        return new IIRow(key, value, null);
 	}
 
-	ImmutableBytesWritable encodeKey(short shard, long timestamp, int col,
-			int colValue) {
+	ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) {
 		byte[] bytes = new byte[20];
-		int len = encodeKey(shard, timestamp, col, colValue, bytes, 0);
+		int len = encodeKey(shard, timestamp, col, bytes, 0);
 		return new ImmutableBytesWritable(bytes, 0, len);
 	}
 
-	int encodeKey(short shard, long timestamp, int col, int colValue,
-			byte[] buf, int offset) {
+	int encodeKey(short shard, long timestamp, int col, byte[] buf, int offset) {
 		int i = offset;
 
 		BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
@@ -103,55 +83,72 @@ public class IIKeyValueCodec {
 		BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
 		i += COLNO_LEN;
 
-		if (colValue >= 0) {
-			int colLen = infoDigest.length(col);
-			BytesUtil.writeUnsigned(colValue, buf, i, colLen);
-			i += colLen;
-		}
-
 		return i - offset;
 	}
 
-	public Iterable<Slice> decodeKeyValue(
-			Iterable<KeyValuePair> kvs) {
-		return new Decoder(infoDigest, kvs);
+    @Override
+	public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
+		return new Decoder(kvs);
 	}
 
+    private static class SliceIterator implements Iterator<Slice> {
+
+        private final Iterator<IIRow> rowIterator;
+
+        public SliceIterator(Iterator<IIRow> rowIterator) {
+            this.rowIterator = rowIterator;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (!rowIterator.hasNext()) {
+                return false;
+            }
+            return false;
+        }
+
+        @Override
+        public Slice next() {
+            return null;
+        }
+
+        @Override
+        public void remove() {
+
+        }
+    }
+
 	private static class Decoder implements Iterable<Slice> {
 
 		TableRecordInfoDigest info;
-		Iterator<KeyValuePair> iterator;
+		Iterator<IIRow> iterator;
 
-		Slice next = null;
+		Slice slice = null;
 		short curShard = Short.MIN_VALUE;
 		long curSliceTimestamp = Long.MIN_VALUE;
 		int curCol = -1;
-		int curColValue = -1;
 		short lastShard = Short.MIN_VALUE;
 		long lastSliceTimestamp = Long.MIN_VALUE;
 		int lastCol = -1;
 		ColumnValueContainer[] containers = null;
-		List<ImmutableBytesWritable> bitMapValues = Lists.newArrayList();
         Map<Integer, Dictionary<?>> localDictionaries = Maps.newHashMap();
 
-		Decoder(TableRecordInfoDigest info,
-				Iterable<KeyValuePair> kvs) {
-			this.info = info;
+		Decoder(Iterable<IIRow> kvs) {
 			this.iterator = kvs.iterator();
 		}
 
 		private void goToNext() {
-			if (next != null) { // was not fetched
+			if (slice != null) { // was not fetched
 				return;
 			}
 
 			// NOTE the input keys are ordered
-			while (next == null && iterator.hasNext()) {
-                KeyValuePair kv = iterator.next();
+			while (slice == null && iterator.hasNext()) {
+                IIRow kv = iterator.next();
 				ImmutableBytesWritable k = kv.getKey();
 				ImmutableBytesWritable v = kv.getValue();
 				decodeKey(k);
-                localDictionaries.put(curCol, kv.getDictionary());
+//                localDictionaries.put(curCol, kv.getDictionary());
 
 				if (curShard != lastShard
 						|| curSliceTimestamp != lastSliceTimestamp) {
@@ -159,7 +156,7 @@ public class IIKeyValueCodec {
 				}
 				consumeCurrent(v);
 			}
-			if (next == null) {
+			if (slice == null) {
 				makeNext();
 			}
 		}
@@ -176,63 +173,26 @@ public class IIKeyValueCodec {
 			curCol = BytesUtil.readUnsigned(buf, i, COLNO_LEN);
 			i += COLNO_LEN;
 
-			if (i - k.getOffset() < k.getLength()) {
-				// bitmap
-				int colLen = info.length(curCol);
-				curColValue = BytesUtil.readUnsigned(buf, i, colLen);
-				i += colLen;
-			} else {
-				// value list
-				curColValue = -1;
-			}
 		}
 
 		private void consumeCurrent(ImmutableBytesWritable v) {
-			if (curCol != lastCol && bitMapValues.size() > 0) { // end of a
-																// bitmap
-																// container
-				addBitMapContainer(lastCol);
-			}
-			if (curColValue < 0) {
-				CompressedValueContainer c = new CompressedValueContainer(info,
-						curCol, 0);
-				c.fromBytes(v);
-				addContainer(curCol, c);
-			} else {
-				assert curColValue == bitMapValues.size();
-				// make a copy, the value object from caller is typically reused
-				// through iteration
-				bitMapValues.add(new ImmutableBytesWritable(v));
-			}
-
 			lastShard = curShard;
 			lastSliceTimestamp = curSliceTimestamp;
 			lastCol = curCol;
 		}
 
 		private void makeNext() {
-			if (bitMapValues.isEmpty() == false) {
-				addBitMapContainer(lastCol);
-			}
 			if (containers != null) {
-				next = new Slice(info, lastShard, lastSliceTimestamp,
+				slice = new Slice(info, lastShard, lastSliceTimestamp,
 						containers);
-                next.setLocalDictionaries(Maps.newHashMap(localDictionaries));
+                slice.setLocalDictionaries(Maps.newHashMap(localDictionaries));
 			}
 			lastSliceTimestamp = Long.MIN_VALUE;
 			lastCol = -1;
 			containers = null;
-			bitMapValues.clear();
             localDictionaries.clear();
 		}
 
-		private void addBitMapContainer(int col) {
-			BitMapContainer c = new BitMapContainer(info, col);
-			c.fromBytes(bitMapValues);
-			addContainer(col, c);
-			bitMapValues.clear();
-		}
-
 		private void addContainer(int col, ColumnValueContainer c) {
 			if (containers == null) {
 				containers = new ColumnValueContainer[info.getColumnCount()];
@@ -246,13 +206,13 @@ public class IIKeyValueCodec {
 				@Override
 				public boolean hasNext() {
 					goToNext();
-					return next != null;
+					return slice != null;
 				}
 
 				@Override
 				public Slice next() {
-					Slice result = next;
-					next = null;
+					Slice result = slice;
+					slice = null;
 					return result;
 				}
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
new file mode 100644
index 0000000..aba4fff
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
@@ -0,0 +1,67 @@
+/*
+ *
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ *  contributor license agreements. See the NOTICE file distributed with
+ *
+ *  this work for additional information regarding copyright ownership.
+ *
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ *  (the "License"); you may not use this file except in compliance with
+ *
+ *  the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *
+ *  limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+
+/**
+ * Created by qianzhou on 3/10/15.
+ */
+public final class IIRow {
+
+    private final ImmutableBytesWritable key;
+    private final ImmutableBytesWritable value;
+    private final ImmutableBytesWritable dictionary;
+
+    public IIRow(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) {
+        this.key = key;
+        this.value = value;
+        this.dictionary = dictionary;
+    }
+    public IIRow() {
+        this(new ImmutableBytesWritable(), new ImmutableBytesWritable(), new ImmutableBytesWritable());
+    }
+
+    public ImmutableBytesWritable getKey() {
+        return key;
+    }
+
+    public ImmutableBytesWritable getValue() {
+        return value;
+    }
+    public ImmutableBytesWritable getDictionary() {
+        return dictionary;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
new file mode 100644
index 0000000..bbd8d61
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
@@ -0,0 +1,49 @@
+/*
+ *
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ *  contributor license agreements. See the NOTICE file distributed with
+ *
+ *  this work for additional information regarding copyright ownership.
+ *
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ *  (the "License"); you may not use this file except in compliance with
+ *
+ *  the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ *  See the License for the specific language governing permissions and
+ *
+ *  limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.invertedindex.model;
+
+import org.apache.kylin.invertedindex.index.Slice;
+
+import java.util.Collection;
+
+/**
+ * Created by qianzhou on 3/12/15.
+ */
+public interface KeyValueCodec {
+
+    Collection<IIRow> encodeKeyValue(Slice slice);
+
+    Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs);
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
deleted file mode 100644
index 48c45f6..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValuePair.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- *
- *
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *
- *  contributor license agreements. See the NOTICE file distributed with
- *
- *  this work for additional information regarding copyright ownership.
- *
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *
- *  (the "License"); you may not use this file except in compliance with
- *
- *  the License. You may obtain a copy of the License at
- *
- *
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- *  Unless required by applicable law or agreed to in writing, software
- *
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- *  See the License for the specific language governing permissions and
- *
- *  limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.dict.Dictionary;
-
-/**
- * Created by qianzhou on 3/10/15.
- */
-public final class KeyValuePair {
-
-    private final ImmutableBytesWritable key;
-    private final ImmutableBytesWritable value;
-    private final int id;
-    private Dictionary<?> dictionary;
-
-    public KeyValuePair(ImmutableBytesWritable key, ImmutableBytesWritable value) {
-        this(-1, key, value);
-    }
-    public KeyValuePair(int id, ImmutableBytesWritable key, ImmutableBytesWritable value) {
-        this.id = id;
-        this.key = key;
-        this.value = value;
-    }
-
-    public void setDictionary(Dictionary<?> dictionary) {
-        this.dictionary = dictionary;
-    }
-
-    public int getId() {
-        return this.id;
-    }
-
-    public Dictionary<?> getDictionary() {
-        return dictionary;
-    }
-
-    public ImmutableBytesWritable getKey() {
-        return key;
-    }
-
-    public ImmutableBytesWritable getValue() {
-        return value;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
index eb65ca8..aecc37d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/tools/IICLI.java
@@ -33,7 +33,7 @@ import org.apache.kylin.invertedindex.index.Slice;
 import org.apache.kylin.invertedindex.index.TableRecord;
 import org.apache.kylin.invertedindex.index.TableRecordInfo;
 import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -54,7 +54,7 @@ public class IICLI {
 		System.out.println("Reading from " + path + " ...");
 
 		TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment());
-		IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+		IIKeyValueCodec codec = new IIKeyValueCodec();
 		int count = 0;
 		for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) {
 			for (RawTableRecord rec : slice) {
@@ -65,17 +65,17 @@ public class IICLI {
 		System.out.println("Total " + count + " records");
 	}
 
-	public static Iterable<KeyValuePair> readSequenceKVs(
+	public static Iterable<IIRow> readSequenceKVs(
 			Configuration hconf, String path) throws IOException {
 		final Reader reader = new Reader(hconf,
 				SequenceFile.Reader.file(new Path(path)));
-		return new Iterable<KeyValuePair>() {
+		return new Iterable<IIRow>() {
 			@Override
-			public Iterator<KeyValuePair> iterator() {
-				return new Iterator<KeyValuePair>() {
+			public Iterator<IIRow> iterator() {
+				return new Iterator<IIRow>() {
 					ImmutableBytesWritable k = new ImmutableBytesWritable();
 					ImmutableBytesWritable v = new ImmutableBytesWritable();
-                    KeyValuePair pair = new KeyValuePair(k, v);
+                    IIRow pair = new IIRow(k, v, null);
 
 					@Override
 					public boolean hasNext() {
@@ -93,7 +93,7 @@ public class IICLI {
 					}
 
 					@Override
-					public KeyValuePair next() {
+					public IIRow next() {
 						return pair;
 					}
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
index f040820..b152cdd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
@@ -29,8 +29,7 @@ import org.apache.kylin.invertedindex.index.*;
 import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -63,33 +62,6 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
 	}
 
 	@Test
-	public void testBitMapContainer() {
-		// create container
-		BitMapContainer container = new BitMapContainer(info.getDigest(), 0);
-		Dictionary<String> dict = info.dict(0);
-		for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
-			container.append(v);
-		}
-		container.append(Dictionary.NULL_ID[dict.getSizeOfId()]);
-		container.closeForChange();
-
-		// copy by serialization
-		List<ImmutableBytesWritable> bytes = container.toBytes();
-		BitMapContainer container2 = new BitMapContainer(info.getDigest(), 0);
-		container2.fromBytes(bytes);
-
-		// check the copy
-		int i = 0;
-		for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
-			int value = container2.getValueIntAt(i++);
-			assertEquals(v, value);
-		}
-		assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()],
-				container2.getValueIntAt(i++));
-		assertEquals(container, container2);
-	}
-
-	@Test
 	public void testCompressedValueContainer() {
 		// create container
 		CompressedValueContainer container = new CompressedValueContainer(
@@ -136,8 +108,8 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
 		List<Slice> slices = buildTimeSlices(records);
 		System.out.println(slices.size() + " slices");
 
-		IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
-		List<KeyValuePair> kvs = encodeKVs(codec, slices);
+		IIKeyValueCodec codec = new IIKeyValueCodec();
+		List<IIRow> kvs = encodeKVs(codec, slices);
 		System.out.println(kvs.size() + " KV pairs");
 
 		List<Slice> slicesCopy = decodeKVs(codec, kvs);
@@ -198,10 +170,10 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
 		return slices;
 	}
 
-	private List<KeyValuePair> encodeKVs(
+	private List<IIRow> encodeKVs(
 			IIKeyValueCodec codec, List<Slice> slices) {
 
-		List<KeyValuePair> kvs = Lists
+		List<IIRow> kvs = Lists
 				.newArrayList();
 		for (Slice slice : slices) {
 			kvs.addAll(codec.encodeKeyValue(slice));
@@ -210,7 +182,7 @@ public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
 	}
 
 	private List<Slice> decodeKVs(IIKeyValueCodec codec,
-			List<KeyValuePair> kvs) {
+			List<IIRow> kvs) {
 		List<Slice> slices = Lists.newArrayList();
 		for (Slice slice : codec.decodeKeyValue(kvs)) {
 			slices.add(slice);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
index 40cca36..285b058 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexReducer.java
@@ -31,7 +31,7 @@ import org.apache.kylin.invertedindex.index.SliceBuilder;
 import org.apache.kylin.invertedindex.index.TableRecord;
 import org.apache.kylin.invertedindex.index.TableRecordInfo;
 import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
 import org.apache.kylin.job.constant.BatchConstants;
 import org.apache.kylin.job.hadoop.AbstractHadoopJob;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -60,7 +60,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
         info = new TableRecordInfo(seg);
         rec = info.createTableRecord();
         builder = null;
-        kv = new IIKeyValueCodec(info.getDigest());
+        kv = new IIKeyValueCodec();
     }
 
     @Override
@@ -92,7 +92,7 @@ public class InvertedIndexReducer extends KylinReducer<LongWritable, ImmutableBy
     }
 
     private void output(Slice slice, Context context) throws IOException, InterruptedException {
-        for (KeyValuePair pair : kv.encodeKeyValue(slice)) {
+        for (IIRow pair : kv.encodeKeyValue(slice)) {
             context.write(pair.getKey(), pair.getValue());
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
index 29b2bcf..05d2421 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/HBaseClientKVIterator.java
@@ -29,15 +29,14 @@ import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
 import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
 
 /**
  * @author yangli9
  * 
  */
-public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable {
+public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
 
     byte[] family;
 
@@ -60,15 +59,15 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
     }
 
     @Override
-    public Iterator<KeyValuePair> iterator() {
+    public Iterator<IIRow> iterator() {
         return new MyIterator();
     }
 
-    private class MyIterator implements Iterator<KeyValuePair> {
+    private class MyIterator implements Iterator<IIRow> {
 
         ImmutableBytesWritable key = new ImmutableBytesWritable();
         ImmutableBytesWritable value = new ImmutableBytesWritable();
-        KeyValuePair pair = new KeyValuePair(key, value);
+        IIRow pair = new IIRow(key, value);
 
         @Override
         public boolean hasNext() {
@@ -76,7 +75,7 @@ public class HBaseClientKVIterator implements Iterable<KeyValuePair>, Closeable
         }
 
         @Override
-        public KeyValuePair next() {
+        public IIRow next() {
             Result r = iterator.next();
             Cell c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
             key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
index 47aa7d9..3dbc8ea 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/HbaseServerKVIterator.java
@@ -18,21 +18,17 @@
 
 package org.apache.kylin.storage.hbase.coprocessor.endpoint;
 
+import com.google.common.collect.Lists;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
 import java.io.Closeable;
-import java.io.DataInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -41,13 +37,11 @@ import java.util.List;
 /**
  * Created by honma on 11/10/14.
  */
-public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable {
+public class HbaseServerKVIterator implements Iterable<IIRow>, Closeable {
 
     private RegionScanner innerScanner;
     private Logger logger = LoggerFactory.getLogger(HbaseServerKVIterator.class);
 
-    List<Cell> results = new ArrayList<Cell>();
-
     public HbaseServerKVIterator(RegionScanner innerScanner) {
         this.innerScanner = innerScanner;
     }
@@ -57,69 +51,55 @@ public class HbaseServerKVIterator implements Iterable<KeyValuePair>, Closeable
         IOUtils.closeQuietly(this.innerScanner);
     }
 
-    @Override
-    public Iterator<KeyValuePair> iterator() {
-        return new Iterator<KeyValuePair>() {
+    private static class IIRowIterator implements Iterator<IIRow> {
 
-            ImmutableBytesWritable key = new ImmutableBytesWritable();
-            ImmutableBytesWritable value = new ImmutableBytesWritable();
-            ImmutableBytesWritable dict = new ImmutableBytesWritable();
-            KeyValuePair pair = new KeyValuePair(key, value);
+        private final RegionScanner regionScanner;
+        private final IIRow row = new IIRow();
+        private boolean hasMore = true;
+        List<Cell> results = Lists.newArrayList();
 
-            private boolean hasMore = true;
+        IIRowIterator(RegionScanner innerScanner) {
+            this.regionScanner = innerScanner;
+        }
 
-            @Override
-            public boolean hasNext() {
-                return hasMore;
-            }
 
-            @Override
-            public KeyValuePair next() {
-                if (hasNext()) {
-                    try {
-                        hasMore = innerScanner.nextRaw(results);
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-
-                    if (results.size() < 1)
-                        throw new IllegalStateException("Hbase row contains less than 1 cell");
-
-                    Dictionary<?> dictionary = null;
-                    boolean hasDictData = false;
-                    for (Cell c : results) {
-                        if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
-                            key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
-                            value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
-                        } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
-                            dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
-                            hasDictData = true;
-                        }
-                    }
-                    if (hasDictData) {
-                        try {
-                            final DataInputStream in = new DataInputStream(new ByteArrayInputStream(dict.get(), dict.getOffset(), dict.getLength()));
-                            String type = in.readUTF();
-                            dictionary = (Dictionary<?>) ClassUtil.forName(type, Dictionary.class).newInstance();
-                            dictionary.readFields(in);
-                        } catch (Exception e) {
-                            logger.error("error create dictionary", e);
-                        }
-                    }
-                    pair.setDictionary(dictionary);
-
-                    results.clear();
-                    return pair;
-                } else {
-                    return null;
+        @Override
+        public boolean hasNext() {
+            return hasMore;
+        }
+
+        @Override
+        public IIRow next() {
+            results.clear();
+            try {
+                hasMore = regionScanner.next(results);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            if (results.size() < 1) {
+                throw new IllegalStateException("Hbase row contains less than 1 cell");
+            }
+            for (Cell c : results) {
+                if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
+                    row.getKey().set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
+                    row.getValue().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
+                } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
+                    row.getDictionary().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
                 }
             }
+            return row;
+        }
 
-            @Override
-            public void remove() {
-                throw new UnsupportedOperationException();
-            }
-        };
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+
+    @Override
+    public Iterator<IIRow> iterator() {
+        return new IIRowIterator(innerScanner);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index fbe3e8b..a20104d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -22,19 +22,7 @@ import com.google.protobuf.ByteString;
 import com.google.protobuf.RpcCallback;
 import com.google.protobuf.RpcController;
 import com.google.protobuf.Service;
-
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
-import org.apache.kylin.storage.hbase.coprocessor.*;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
-
 import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
@@ -45,7 +33,15 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.protobuf.ResponseConverter;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.storage.filter.BitMapFilterEvaluator;
+import org.apache.kylin.storage.hbase.coprocessor.*;
+import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -94,7 +90,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
             region.startRegionOperation();
 
             synchronized (innerScanner) {
-                IIKeyValueCodec codec = new IIKeyValueCodec(tableRecordInfoDigest);
+                IIKeyValueCodec codec = new IIKeyValueCodec();
                 //TODO pass projector to codec to skip loading columns
                 Iterable<Slice> slices = codec.decodeKeyValue(new HbaseServerKVIterator(innerScanner));
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab458df4/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 5ed48cf..10d212a 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Pair;
 import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.invertedindex.index.Slice;
@@ -51,7 +50,7 @@ import org.apache.kylin.invertedindex.index.TableRecord;
 import org.apache.kylin.invertedindex.index.TableRecordInfo;
 import org.apache.kylin.invertedindex.model.IIDesc;
 import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.KeyValuePair;
+import org.apache.kylin.invertedindex.model.IIRow;
 import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -104,7 +103,7 @@ public class IIStreamBuilder extends StreamBuilder {
                 return parseStream(input, desc);
             }
         });
-        final Map<TblColRef, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
+        final Map<Integer, Dictionary<?>> dictionaryMap = buildDictionary(table, desc);
         Map<TblColRef, FixedLenMeasureCodec<?>> measureCodecMap = Maps.newHashMap();
         int index = 0;
         for (TblColRef tblColRef : desc.listAllColumns()) {
@@ -115,14 +114,14 @@ public class IIStreamBuilder extends StreamBuilder {
         }
         TableRecordInfo tableRecordInfo = new TableRecordInfo(desc, dictionaryMap, measureCodecMap);
         SliceBuilder sliceBuilder = new SliceBuilder(tableRecordInfo, (short) partitionId);
-        final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo);
-        loadToHBase(hTable, slice, new IIKeyValueCodec(tableRecordInfo.getDigest()), desc.listAllColumns(), dictionaryMap);
+        final Slice slice = buildSlice(table, sliceBuilder, tableRecordInfo, dictionaryMap);
+        loadToHBase(hTable, slice, new IIKeyValueCodec());
         submitOffset();
         stopwatch.stop();
         logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + TimeUnit.MILLISECONDS);
     }
 
-    private Map<TblColRef, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
+    private Map<Integer, Dictionary<?>> buildDictionary(List<List<String>> table, IIDesc desc) {
         SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
         Set<TblColRef> dimensionColumns = Sets.newHashSet();
         for (int i = 0; i < desc.listAllColumns().size(); i++) {
@@ -136,9 +135,9 @@ public class IIStreamBuilder extends StreamBuilder {
                 valueMap.put(desc.listAllColumns().get(i), cell);
             }
         }
-        Map<TblColRef, Dictionary<?>> result = Maps.newHashMap();
+        Map<Integer, Dictionary<?>> result = Maps.newHashMap();
         for (TblColRef tblColRef : valueMap.keys()) {
-            result.put(tblColRef, DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
+            result.put(desc.findColumn(tblColRef), DictionaryGenerator.buildDictionaryFromValueList(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
                 @Nullable
                 @Override
                 public byte[] apply(String input) {
@@ -153,7 +152,7 @@ public class IIStreamBuilder extends StreamBuilder {
         return Lists.newArrayList(new String(stream.getRawData()).split(","));
     }
 
-    private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo) {
+    private Slice buildSlice(List<List<String>> table, SliceBuilder sliceBuilder, TableRecordInfo tableRecordInfo, Map<Integer, Dictionary<?>> localDictionary) {
         for (List<String> row : table) {
             TableRecord tableRecord = tableRecordInfo.createTableRecord();
             for (int i = 0; i < row.size(); i++) {
@@ -161,31 +160,35 @@ public class IIStreamBuilder extends StreamBuilder {
             }
             sliceBuilder.append(tableRecord);
         }
-        return sliceBuilder.close();
+        final Slice slice = sliceBuilder.close();
+        slice.setLocalDictionaries(localDictionary);
+        return slice;
     }
 
-    private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec, List<TblColRef> allColumns, Map<TblColRef, Dictionary<?>> localDictionaries) throws IOException {
+    private byte[] getDictBytes(Dictionary<?> dict) throws IOException {
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        DataOutputStream out = new DataOutputStream(baos);
+        out.writeUTF(dict.getClass().getName());
+        dict.write(out);
+        return baos.toByteArray();
+    }
+
+    private void loadToHBase(HTableInterface hTable, Slice slice, IIKeyValueCodec codec) throws IOException {
         try {
             List<Put> data = Lists.newArrayList();
-            for (KeyValuePair pair : codec.encodeKeyValue(slice)) {
-                final byte[] key = pair.getKey().get();
-                final byte[] value = pair.getValue().get();
+            for (IIRow row : codec.encodeKeyValue(slice)) {
+                final byte[] key = row.getKey().get();
+                final byte[] value = row.getValue().get();
                 Put put = new Put(key);
                 put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, value);
-
-                //dictionary
-                final int id = pair.getId();
-                if (id >= 0 && id < allColumns.size()) {
-                    final Dictionary<?> dictionary = localDictionaries.get(allColumns.get(id));
-                    if (dictionary != null) {
-                        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-                        DataOutputStream out = new DataOutputStream(baos);
-                        out.writeUTF(dictionary.getClass().getName());
-                        dictionary.write(out);
-                        put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, baos.toByteArray());
-                    }
-                    data.add(put);
+                final ImmutableBytesWritable dictionary = row.getDictionary();
+                final byte[] dictBytes = dictionary.get();
+                if (dictionary.getOffset() == 0 && dictionary.getLength() == dictBytes.length) {
+                    put.add(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, dictBytes);
+                } else {
+                    throw new RuntimeException("dict offset should be 0, and dict length should be " + dictBytes.length + " but they are" + dictionary.getOffset() + " " + dictionary.getLength());
                 }
+                data.add(put);
             }
             hTable.put(data);
         } finally {