You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/26 16:08:39 UTC

[16/34] incubator-kylin git commit: KYLIN-960 organize existing storage module

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
deleted file mode 100644
index 2eba0e3..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.tuple.TupleInfo;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * @author xjiang
- */
-public class SerializedHBaseTupleIterator implements ITupleIterator {
-
-    private static final int PARTIAL_DEFAULT_LIMIT = 10000;
-
-    private final StorageContext context;
-    private final int partialResultLimit;
-    private final List<CubeSegmentTupleIterator> segmentIteratorList;
-    private final Iterator<CubeSegmentTupleIterator> segmentIteratorIterator;
-
-    private ITupleIterator segmentIterator;
-    private int scanCount;
-    private ITuple next;
-
-    public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
-            Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
-            StorageContext context, TupleInfo returnTupleInfo) {
-
-        this.context = context;
-        int limit = context.getLimit();
-        this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);
-
-        this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
-        Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
-        for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
-            CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
-            this.segmentIteratorList.add(segIter);
-        }
-
-        this.segmentIteratorIterator = this.segmentIteratorList.iterator();
-        if (this.segmentIteratorIterator.hasNext()) {
-            this.segmentIterator = this.segmentIteratorIterator.next();
-        } else {
-            this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
-        }
-    }
-
-    private Map<CubeSegment, List<HBaseKeyRange>> makeRangesMap(List<HBaseKeyRange> segmentKeyRanges) {
-        Map<CubeSegment, List<HBaseKeyRange>> map = Maps.newHashMap();
-        for (HBaseKeyRange range : segmentKeyRanges) {
-            List<HBaseKeyRange> list = map.get(range.getCubeSegment());
-            if (list == null) {
-                list = Lists.newArrayList();
-                map.put(range.getCubeSegment(), list);
-            }
-            list.add(range);
-        }
-        return map;
-    }
-
-    @Override
-    public boolean hasNext() {
-        if (next != null)
-            return true;
-
-        // 1. check limit
-        if (context.isLimitEnabled() && scanCount >= context.getLimit()) {
-            return false;
-        }
-        // 2. check partial result
-        if (context.isAcceptPartialResult() && scanCount > partialResultLimit) {
-            context.setPartialResultReturned(true);
-            return false;
-        }
-        // 3. check threshold
-        if (scanCount >= context.getThreshold()) {
-            throw new ScanOutOfLimitException("Scan row count exceeded threshold: " + context.getThreshold() + ", please add filter condition to narrow down backend scan range, like where clause.");
-        }
-        // 4. check cube segments
-        if (segmentIterator.hasNext()) {
-            next = segmentIterator.next();
-            scanCount++;
-            return true;
-        } else if (segmentIteratorIterator.hasNext()) {
-            segmentIterator.close();
-            segmentIterator = segmentIteratorIterator.next();
-            return hasNext();
-        }
-        return false;
-    }
-
-    @Override
-    public ITuple next() {
-        if (next == null) {
-            hasNext();
-            if (next == null)
-                throw new NoSuchElementException();
-        }
-        ITuple r = next;
-        next = null;
-        return r;
-    }
-
-    @Override
-    public void remove() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void close() {
-        context.setTotalScanCount(scanCount);
-
-        // hasNext() loop may exit because of limit, threshold, etc.
-        // close all the remaining segmentIterator
-        segmentIterator.close();
-        while (segmentIteratorIterator.hasNext()) {
-            segmentIterator = segmentIteratorIterator.next();
-            segmentIterator.close();
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
deleted file mode 100644
index 9e8c4a0..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Arrays;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.kv.RowConstants;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class AggrKey implements Comparable<AggrKey> {
-
-    final byte[] groupByMask;
-    final transient int[] groupByMaskSet;
-    transient int hashcode;
-    byte[] data;
-    int offset;
-
-    AggrKey(byte[] groupByMask) {
-        this.groupByMask = groupByMask;
-        LinkedList<Integer> list = Lists.newLinkedList();
-        for (int i = 0; i < groupByMask.length; i++) {
-            if (groupByMask[i] != 0) {
-                list.add(i);
-            }
-        }
-        groupByMaskSet = new int[list.size()];
-        for (int i = 0; i < list.size(); i++) {
-            groupByMaskSet[i] = list.get(i);
-        }
-    }
-
-    private AggrKey(byte[] groupByMask, int[] groupByMaskSet, byte[] data, int offset, int hashcode) {
-        this.groupByMask = groupByMask;
-        this.groupByMaskSet = groupByMaskSet;
-        this.data = data;
-        this.offset = offset;
-        this.hashcode = hashcode;
-    }
-
-    private int calculateHash() {
-        int hash = 1;
-        for (int i = 0; i < groupByMaskSet.length; i++) {
-            byte t = data[offset + groupByMaskSet[i]];
-            if (t != RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
-                hash = (31 * hash) + t;
-            }
-        }
-        return hash;
-    }
-
-    public byte[] get() {
-        return data;
-    }
-
-    public int offset() {
-        return offset;
-    }
-
-    public int length() {
-        return groupByMask.length;
-    }
-
-    void set(byte[] data, int offset) {
-        this.data = data;
-        this.offset = offset;
-        this.hashcode = calculateHash();
-    }
-
-    public byte[] getGroupByMask() {
-        return this.groupByMask;
-    }
-
-    public byte[] copyBytes() {
-        return Arrays.copyOfRange(data, offset, offset + length());
-    }
-
-    AggrKey copy() {
-        AggrKey copy = new AggrKey(this.groupByMask, this.groupByMaskSet, copyBytes(), 0, this.hashcode);
-        return copy;
-    }
-
-    @Override
-    public int hashCode() {
-        return hashcode;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj)
-            return true;
-        if (obj == null)
-            return false;
-        if (getClass() != obj.getClass())
-            return false;
-        AggrKey other = (AggrKey) obj;
-        return compareTo(other) == 0;
-    }
-
-    @Override
-    public int compareTo(AggrKey o) {
-        int comp = this.length() - o.length();
-        if (comp != 0)
-            return comp;
-
-        for (int i = 0; i < groupByMaskSet.length; i++) {
-            comp = BytesUtil.compareByteUnsigned(this.data[this.offset + groupByMaskSet[i]], o.data[o.offset + groupByMaskSet[i]]);
-            if (comp != 0)
-                return comp;
-        }
-        return 0;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
deleted file mode 100644
index b7cee9f..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Map;
-
-import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-
-import com.google.common.collect.Maps;
-
-/**
- */
-@SuppressWarnings("rawtypes")
-public abstract class AggregationCache {
-    static final long MEMORY_USAGE_CAP = 500 * 1024 * 1024; // 500 MB
-    static final long MEMOERY_MAX_BYTES = Runtime.getRuntime().maxMemory();
-    protected final Map<AggrKey, MeasureAggregator[]> aggBufMap;
-    transient int rowMemBytes;
-    private AggrKey firstKey = null;
-
-    public AggregationCache() {
-        this.aggBufMap = Maps.newHashMap();
-    }
-
-    public abstract MeasureAggregator[] createBuffer();
-
-    public MeasureAggregator[] getBuffer(AggrKey aggkey) {
-        MeasureAggregator[] aggBuf = aggBufMap.get(aggkey);
-        if (aggBuf == null) {
-            aggBuf = createBuffer();
-            AggrKey key = aggkey.copy();
-            aggBufMap.put(key, aggBuf);
-
-            if (firstKey == null) {
-                firstKey = key;
-            }
-        }
-        return aggBuf;
-    }
-
-    public long getSize() {
-        return aggBufMap.size();
-    }
-
-    public void checkMemoryUsage() {
-        if (firstKey == null)
-            return;
-
-        // about memory calculation,
-        // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/
-        if (rowMemBytes <= 0) {
-            if (aggBufMap.size() > 0) {
-                rowMemBytes = 0;
-                MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey);
-                for (MeasureAggregator agg : measureAggregators) {
-                    rowMemBytes += agg.getMemBytesEstimate();
-                }
-            }
-        }
-        int size = aggBufMap.size();
-        long memUsage = (40 + rowMemBytes) * size;
-        if (memUsage > MEMORY_USAGE_CAP) {
-            throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor.");
-        }
-
-        //If less than 5% of max memory
-        long avail = MemoryBudgetController.getSystemAvailBytes();
-        if (avail < (MEMOERY_MAX_BYTES / 20)) {
-            throw new RuntimeException("Running Kylin coprocessor when too little memory is left. Abort coprocessor. Current available memory is " + avail + ". Max memory is " + MEMOERY_MAX_BYTES);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
deleted file mode 100644
index 6e69827..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-/**
- */
-public class CoprocessorConstants {
-    public static final int SERIALIZE_BUFFER_SIZE = 65536;
-    public static final int METRIC_SERIALIZE_BUFFER_SIZE = 65536;
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
deleted file mode 100644
index feb36ca..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Set;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-/**
- * @author yangli9
- */
-public class CoprocessorFilter {
-
-    public static CoprocessorFilter fromFilter(final IDictionaryAware dictionaryAware, TupleFilter rootFilter, FilterDecorator.FilterConstantsTreatment filterConstantsTreatment) {
-        // translate constants into dictionary IDs via a serialize copy
-        FilterDecorator filterDecorator = new FilterDecorator(dictionaryAware, filterConstantsTreatment);
-        byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
-        TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
-
-        return new CoprocessorFilter(copy, filterDecorator.getInevaluableColumns());
-    }
-
-    public static byte[] serialize(CoprocessorFilter o) {
-        return (o.filter == null) ? BytesUtil.EMPTY_BYTE_ARRAY : TupleFilterSerializer.serialize(o.filter, DictCodeSystem.INSTANCE);
-    }
-
-    public static CoprocessorFilter deserialize(byte[] filterBytes) {
-        TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null //
-                : TupleFilterSerializer.deserialize(filterBytes, DictCodeSystem.INSTANCE);
-        return new CoprocessorFilter(filter, null);
-    }
-
-    // ============================================================================
-
-    private final TupleFilter filter;
-    private final Set<TblColRef> inevaluableColumns;
-
-    public CoprocessorFilter(TupleFilter filter, Set<TblColRef> inevaluableColumns) {
-        this.filter = filter;
-        this.inevaluableColumns = inevaluableColumns;
-    }
-
-    public TupleFilter getFilter() {
-        return filter;
-    }
-
-    public Set<TblColRef> getInevaluableColumns() {
-        return inevaluableColumns;
-    }
-
-    public boolean evaluate(IEvaluatableTuple tuple) {
-        if (filter == null)
-            return true;
-        else
-            return filter.evaluate(tuple, DictCodeSystem.INSTANCE);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
deleted file mode 100644
index 2821a6c..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowKeyEncoder;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author yangli9
- */
-public class CoprocessorProjector {
-
-    public static CoprocessorProjector makeForObserver(final CubeSegment cubeSegment, final Cuboid cuboid, final Collection<TblColRef> dimensionColumns) {
-
-        RowKeyEncoder rowKeyMaskEncoder = new RowKeyEncoder(cubeSegment, cuboid) {
-            @Override
-            protected int fillHeader(byte[] bytes, byte[][] values) {
-                Arrays.fill(bytes, 0, this.headerLength, (byte) 0xff);
-                return this.headerLength;
-            }
-
-            @Override
-            protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) {
-                byte bits = dimensionColumns.contains(column) ? (byte) 0xff : 0x00;
-                Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, bits);
-            }
-        };
-
-        byte[] mask = rowKeyMaskEncoder.encode(new byte[cuboid.getColumns().size()][]);
-        return new CoprocessorProjector(mask, dimensionColumns.size() != 0);
-    }
-
-    public static CoprocessorProjector makeForEndpoint(final TableRecordInfo tableInfo, final Collection<TblColRef> groupby) {
-        byte[] mask = new byte[tableInfo.getDigest().getByteFormLen()];
-        int maskIdx = 0;
-        for (int i = 0; i < tableInfo.getDigest().getColumnCount(); ++i) {
-            TblColRef tblColRef = tableInfo.getColumns().get(i);
-            int length = tableInfo.getDigest().length(i);
-            byte bits = groupby.contains(tblColRef) ? (byte) 0xff : 0x00;
-            for (int j = 0; j < length; ++j) {
-                mask[maskIdx++] = bits;
-            }
-        }
-        return new CoprocessorProjector(mask, groupby.size() != 0);
-    }
-
-    public static byte[] serialize(CoprocessorProjector o) {
-        ByteBuffer buf = ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
-        serializer.serialize(o, buf);
-        byte[] result = new byte[buf.position()];
-        System.arraycopy(buf.array(), 0, result, 0, buf.position());
-        return result;
-    }
-
-    public static CoprocessorProjector deserialize(byte[] bytes) {
-        return serializer.deserialize(ByteBuffer.wrap(bytes));
-    }
-
-    private static final Serializer serializer = new Serializer();
-
-    private static class Serializer implements BytesSerializer<CoprocessorProjector> {
-
-        @Override
-        public void serialize(CoprocessorProjector value, ByteBuffer out) {
-            BytesUtil.writeByteArray(value.groupByMask, out);
-            BytesUtil.writeVInt(value.hasGroupby ? 1 : 0, out);
-        }
-
-        @Override
-        public CoprocessorProjector deserialize(ByteBuffer in) {
-            byte[] mask = BytesUtil.readByteArray(in);
-            boolean hasGroupBy = BytesUtil.readVInt(in) == 1;
-            return new CoprocessorProjector(mask, hasGroupBy);
-        }
-    }
-
-    // ============================================================================
-
-    final transient AggrKey aggrKey;
-    final byte[] groupByMask; // mask out columns that are not needed (by group by)
-    final boolean hasGroupby;
-
-    public CoprocessorProjector(byte[] groupByMask, boolean hasGroupby) {
-        this.groupByMask = groupByMask;
-        this.aggrKey = new AggrKey(groupByMask);
-        this.hasGroupby = hasGroupby;
-    }
-
-    public boolean hasGroupby() {
-        return hasGroupby;
-    }
-
-    public AggrKey getAggrKey(List<Cell> rowCells) {
-        Cell cell = rowCells.get(0);
-        assert groupByMask.length == cell.getRowLength();
-
-        aggrKey.set(cell.getRowArray(), cell.getRowOffset());
-        return aggrKey;
-    }
-
-    public AggrKey getAggrKey(byte[] row) {
-        assert groupByMask.length == row.length;
-        aggrKey.set(row, 0);
-        return aggrKey;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java
deleted file mode 100644
index 11cb4e8..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- */
-public class CoprocessorRowType {
-
-    //for endpoint
-    public static CoprocessorRowType fromTableRecordInfo(TableRecordInfo tableRecordInfo, List<TblColRef> cols) {
-
-        int[] colSizes = new int[cols.size()];
-        for (int i = 0; i < cols.size(); i++) {
-            colSizes[i] = tableRecordInfo.getDigest().length(i);
-        }
-        return new CoprocessorRowType(cols.toArray(new TblColRef[cols.size()]), colSizes);
-    }
-
-    //for observer
-    public static CoprocessorRowType fromCuboid(CubeSegment seg, Cuboid cuboid) {
-        List<TblColRef> colList = cuboid.getColumns();
-        TblColRef[] cols = colList.toArray(new TblColRef[colList.size()]);
-        RowKeyColumnIO colIO = new RowKeyColumnIO(seg);
-        int[] colSizes = new int[cols.length];
-        for (int i = 0; i < cols.length; i++) {
-            colSizes[i] = colIO.getColumnLength(cols[i]);
-        }
-        return new CoprocessorRowType(cols, colSizes);
-    }
-
-    public static byte[] serialize(CoprocessorRowType o) {
-        ByteBuffer buf = ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
-        serializer.serialize(o, buf);
-        byte[] result = new byte[buf.position()];
-        System.arraycopy(buf.array(), 0, result, 0, buf.position());
-        return result;
-    }
-
-    public static CoprocessorRowType deserialize(byte[] bytes) {
-        return serializer.deserialize(ByteBuffer.wrap(bytes));
-    }
-
-    private static final Serializer serializer = new Serializer();
-
-    private static class Serializer implements BytesSerializer<CoprocessorRowType> {
-
-        @Override
-        public void serialize(CoprocessorRowType o, ByteBuffer out) {
-            int n = o.columns.length;
-            BytesUtil.writeVInt(o.columns.length, out);
-            for (int i = 0; i < n; i++) {
-                BytesUtil.writeAsciiString(o.columns[i].getTable(), out);
-                BytesUtil.writeAsciiString(o.columns[i].getName(), out);
-                BytesUtil.writeVInt(o.columnSizes[i], out);
-            }
-        }
-
-        @Override
-        public CoprocessorRowType deserialize(ByteBuffer in) {
-            int n = BytesUtil.readVInt(in);
-            TblColRef[] cols = new TblColRef[n];
-            int[] colSizes = new int[n];
-            for (int i = 0; i < n; i++) {
-                String tableName = BytesUtil.readAsciiString(in);
-                String colName = BytesUtil.readAsciiString(in);
-                TableDesc table = new TableDesc();
-                table.setName(tableName);
-                ColumnDesc col = new ColumnDesc();
-                col.setTable(table);
-                col.setName(colName);
-                col.init(table);
-                cols[i] = new TblColRef(col);
-
-                int colSize = BytesUtil.readVInt(in);
-                colSizes[i] = colSize;
-            }
-            return new CoprocessorRowType(cols, colSizes);
-        }
-    }
-
-    // ============================================================================
-
-    public TblColRef[] columns;
-    public int[] columnSizes;
-    public int[] columnOffsets;
-    public HashMap<TblColRef, Integer> columnIdxMap;
-
-    public CoprocessorRowType(TblColRef[] columns, int[] columnSizes) {
-        this.columns = columns;
-        this.columnSizes = columnSizes;
-        init();
-    }
-
-    public int getColIndexByTblColRef(TblColRef colRef) {
-        return columnIdxMap.get(colRef);
-    }
-
-    private void init() {
-        int[] offsets = new int[columns.length];
-        int o = RowConstants.ROWKEY_CUBOIDID_LEN;
-        for (int i = 0; i < columns.length; i++) {
-            offsets[i] = o;
-            o += columnSizes[i];
-        }
-        this.columnOffsets = offsets;
-
-        HashMap<TblColRef, Integer> map = Maps.newHashMap();
-        for (int i = 0; i < columns.length; i++) {
-            map.put(columns[i], i);
-        }
-        this.columnIdxMap = map;
-    }
-
-    public int getColumnCount() {
-        return columns.length;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
deleted file mode 100644
index d4e2713..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
-
-/**
- * A simple code system where all values are dictionary IDs (fixed length bytes) encoded as ISO-8859-1 strings.
- * 
- * @author yangli9
- */
-public class DictCodeSystem implements IFilterCodeSystem<String> {
-
-    public static final DictCodeSystem INSTANCE = new DictCodeSystem();
-
-    private DictCodeSystem() {
-        // singleton
-    }
-
-    @Override
-    public boolean isNull(String value) {
-        if (value == null)
-            return true;
-
-        String v = value;
-        for (int i = 0, n = v.length(); i < n; i++) {
-            if ((byte) v.charAt(i) != Dictionary.NULL)
-                return false;
-        }
-        return true;
-    }
-
-    @Override
-    public int compare(String tupleValue, String constValue) {
-        return tupleValue.compareTo(constValue);
-    }
-
-    //TODO: should use ISO-8859-1 rather than UTF8
-    @Override
-    public void serialize(String value, ByteBuffer buffer) {
-        BytesUtil.writeUTFString(value, buffer);
-    }
-
-    @Override
-    public String deserialize(ByteBuffer buffer) {
-        return BytesUtil.readUTFString(buffer);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
deleted file mode 100644
index 350b167..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
+++ /dev/null
@@ -1,183 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Collection;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Sets;
-
-/**
- */
-@SuppressWarnings("unchecked")
-public class FilterDecorator implements TupleFilterSerializer.Decorator {
-    public enum FilterConstantsTreatment {
-        AS_IT_IS, REPLACE_WITH_GLOBAL_DICT, REPLACE_WITH_LOCAL_DICT
-    }
-
-    private RowKeyColumnIO columnIO;
-    private Set<TblColRef> inevaluableColumns;
-    private FilterConstantsTreatment filterConstantsTreatment;
-
-    public FilterDecorator(IDictionaryAware seg, FilterConstantsTreatment filterConstantsTreatment) {
-        this.columnIO = new RowKeyColumnIO(seg);
-        this.inevaluableColumns = Sets.newHashSet();
-        this.filterConstantsTreatment = filterConstantsTreatment;
-    }
-
-    public Set<TblColRef> getInevaluableColumns() {
-        return inevaluableColumns;
-    }
-
-    private TupleFilter replaceConstantsWithLocalDict(CompareTupleFilter oldCompareFilter, CompareTupleFilter newCompareFilter) {
-        //TODO localdict: (performance issue) transalte() with roundingflag 0 will use try catch exceptions to deal with non-existing entries
-        return replaceConstantsWithGlobalDict(oldCompareFilter, newCompareFilter);
-    }
-
-    private TupleFilter replaceConstantsWithGlobalDict(CompareTupleFilter oldCompareFilter, CompareTupleFilter newCompareFilter) {
-        Collection<String> constValues = (Collection<String>) oldCompareFilter.getValues();
-        String firstValue = constValues.iterator().next();
-        TblColRef col = newCompareFilter.getColumn();
-
-        TupleFilter result;
-        String v;
-
-        // translate constant into rowkey ID
-        switch (newCompareFilter.getOperator()) {
-        case EQ:
-        case IN:
-            Set<String> newValues = Sets.newHashSet();
-            for (String value : constValues) {
-                v = translate(col, value, 0);
-                if (!isDictNull(v))
-                    newValues.add(v);
-            }
-            if (newValues.isEmpty()) {
-                result = ConstantTupleFilter.FALSE;
-            } else {
-                newCompareFilter.addChild(new ConstantTupleFilter(newValues));
-                result = newCompareFilter;
-            }
-            break;
-        case NEQ:
-            v = translate(col, firstValue, 0);
-            if (isDictNull(v)) {
-                result = ConstantTupleFilter.TRUE;
-            } else {
-                newCompareFilter.addChild(new ConstantTupleFilter(v));
-                result = newCompareFilter;
-            }
-            break;
-        case LT:
-            v = translate(col, firstValue, 1);
-            if (isDictNull(v)) {
-                result = ConstantTupleFilter.TRUE;
-            } else {
-                newCompareFilter.addChild(new ConstantTupleFilter(v));
-                result = newCompareFilter;
-            }
-            break;
-        case LTE:
-            v = translate(col, firstValue, -1);
-            if (isDictNull(v)) {
-                result = ConstantTupleFilter.FALSE;
-            } else {
-                newCompareFilter.addChild(new ConstantTupleFilter(v));
-                result = newCompareFilter;
-            }
-            break;
-        case GT:
-            v = translate(col, firstValue, -1);
-            if (isDictNull(v)) {
-                result = ConstantTupleFilter.TRUE;
-            } else {
-                newCompareFilter.addChild(new ConstantTupleFilter(v));
-                result = newCompareFilter;
-            }
-            break;
-        case GTE:
-            v = translate(col, firstValue, 1);
-            if (isDictNull(v)) {
-                result = ConstantTupleFilter.FALSE;
-            } else {
-                newCompareFilter.addChild(new ConstantTupleFilter(v));
-                result = newCompareFilter;
-            }
-            break;
-        default:
-            throw new IllegalStateException("Cannot handle operator " + newCompareFilter.getOperator());
-        }
-        return result;
-    }
-
-    private boolean isDictNull(String v) {
-        return DictCodeSystem.INSTANCE.isNull(v);
-    }
-
-    @Override
-    public TupleFilter onSerialize(TupleFilter filter) {
-        if (filter == null)
-            return null;
-
-        // In case of NOT(inEvaluatableFilter), we should immediately replace it as TRUE,
-        // Otherwise, inEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
-        // always return FALSE
-        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
-            TupleFilter.collectColumns(filter, inevaluableColumns);
-            return ConstantTupleFilter.TRUE;
-        }
-
-        if (!(filter instanceof CompareTupleFilter))
-            return filter;
-
-        if (!TupleFilter.isEvaluableRecursively(filter)) {
-            TupleFilter.collectColumns(filter, inevaluableColumns);
-            return ConstantTupleFilter.TRUE;
-        }
-
-        if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
-            return filter;
-        } else {
-
-            // extract ColumnFilter & ConstantFilter
-            CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
-            TblColRef col = compareFilter.getColumn();
-
-            if (col == null) {
-                return filter;
-            }
-
-            Collection<String> constValues = (Collection<String>) compareFilter.getValues();
-            if (constValues == null || constValues.isEmpty()) {
-                return filter;
-            }
-
-            CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
-            newCompareFilter.addChild(new ColumnTupleFilter(col));
-
-            if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
-                return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
-            } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
-                return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
-            } else {
-                throw new RuntimeException("should not reach here");
-            }
-        }
-    }
-
-    private String translate(TblColRef column, String v, int roundingFlag) {
-        byte[] value = Bytes.toBytes(v);
-        byte[] id = new byte[columnIO.getColumnLength(column)];
-        columnIO.writeColumn(column, value, value.length, roundingFlag, Dictionary.NULL, id, 0);
-        return Dictionary.dictIdToString(id, 0, id.length);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
deleted file mode 100644
index f2c19d9..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- *
- *
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *
- *  contributor license agreements. See the NOTICE file distributed with
- *
- *  this work for additional information regarding copyright ownership.
- *
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *
- *  (the "License"); you may not use this file except in compliance with
- *
- *  the License. You may obtain a copy of the License at
- *
- *
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- *  Unless required by applicable law or agreed to in writing, software
- *
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- *  See the License for the specific language governing permissions and
- *
- *  limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.util.Map;
-
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
-
-import com.google.common.collect.Maps;
-
-/**
- */
-public class ClearTextDictionary implements IDictionaryAware {
-
-    private final TableRecordInfoDigest digest;
-    private final Map<TblColRef, Integer> columnIndexMap;
-
-    public ClearTextDictionary(TableRecordInfoDigest digest, CoprocessorRowType coprocessorRowType) {
-        this.digest = digest;
-        this.columnIndexMap = coprocessorRowType.columnIdxMap;
-    }
-
-    public ClearTextDictionary(TableRecordInfo tableRecordInfo) {
-        this.digest = tableRecordInfo.getDigest();
-        this.columnIndexMap = Maps.newHashMap();
-        for (int i = 0; i < tableRecordInfo.getColumns().size(); i++) {
-            columnIndexMap.put(tableRecordInfo.getColumns().get(i), i);
-        }
-    }
-
-    @Override
-    public int getColumnLength(TblColRef col) {
-        return digest.length(columnIndexMap.get(col));
-    }
-
-    @Override
-    public Dictionary<?> getDictionary(TblColRef col) {
-        return null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
deleted file mode 100644
index c44ac52..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.protobuf.ResponseConverter;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.CompressionUtils;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.util.KryoUtils;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.gridtable.GTScanRequest;
-import org.apache.kylin.gridtable.IGTScanner;
-import org.apache.kylin.gridtable.IGTStore;
-import org.apache.kylin.storage.cube.CellListIterator;
-import org.apache.kylin.storage.cube.CubeHBaseRPC;
-import org.apache.kylin.storage.cube.HBaseReadonlyStore;
-import org.apache.kylin.storage.cube.RawScan;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.CubeVisitProtos;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Builder;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.Service;
-
-public class CubeVisitService extends CubeVisitProtos.CubeVisitService implements Coprocessor, CoprocessorService {
-
-    private static final Logger logger = LoggerFactory.getLogger(CubeVisitService.class);
-    //TODO limit memory footprint
-    private static final int MEMORY_LIMIT = 500 * 1024 * 1024;
-
-    private RegionCoprocessorEnvironment env;
-
-    private long serviceStartTime;
-
-    static class InnerScannerAsIterator implements CellListIterator {
-        private RegionScanner regionScanner;
-        private List<Cell> nextOne = Lists.newArrayList();
-        private List<Cell> ret = Lists.newArrayList();
-
-        private boolean hasMore;
-
-        public InnerScannerAsIterator(RegionScanner regionScanner) {
-            this.regionScanner = regionScanner;
-
-            try {
-                hasMore = regionScanner.nextRaw(nextOne);
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public boolean hasNext() {
-            return !nextOne.isEmpty();
-        }
-
-        @Override
-        public List<Cell> next() {
-
-            if (nextOne.size() < 1) {
-                throw new IllegalStateException();
-            }
-            ret.clear();
-            ret.addAll(nextOne);
-            nextOne.clear();
-            try {
-                if (hasMore) {
-                    hasMore = regionScanner.nextRaw(nextOne);
-                }
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-            return ret;
-        }
-
-        @Override
-        public void remove() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void close() throws IOException {
-            //does not need to close as regionScanner will be closed in finally block
-        }
-    }
-
-    @Override
-    public void visitCube(RpcController controller, CubeVisitProtos.CubeVisitRequest request, RpcCallback<CubeVisitProtos.CubeVisitResponse> done) {
-        RegionScanner innerScanner = null;
-        HRegion region = null;
-
-        try {
-            this.serviceStartTime = System.currentTimeMillis();
-
-            GTScanRequest scanReq = KryoUtils.deserialize(request.getGtScanRequest().toByteArray(), GTScanRequest.class);
-            RawScan hbaseRawScan = KryoUtils.deserialize(request.getHbaseRawScan().toByteArray(), RawScan.class);
-            //TODO: rewrite own start/end
-            Scan scan = CubeHBaseRPC.buildScan(hbaseRawScan);
-
-            region = env.getRegion();
-            region.startRegionOperation();
-
-            innerScanner = region.getScanner(scan);
-            InnerScannerAsIterator cellListIterator = new InnerScannerAsIterator(innerScanner);
-
-            IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScan.hbaseColumns);
-            IGTScanner rawScanner = store.scan(scanReq);
-            IGTScanner finalScanner = scanReq.decorateScanner(rawScanner);
-
-            ByteBuffer buffer = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-            ByteArrayOutputStream outputStream = new ByteArrayOutputStream(RowConstants.ROWVALUE_BUFFER_SIZE);
-            for (GTRecord oneRecord : finalScanner) {
-                buffer.clear();
-                oneRecord.exportAllColumns(buffer);
-                buffer.flip();
-                outputStream.write(buffer.array(), buffer.arrayOffset() - buffer.position(), buffer.remaining());
-            }
-            //outputStream.close() is not necessary
-            byte[] allRows = outputStream.toByteArray();
-            Builder responseBuilder = CubeVisitProtos.CubeVisitResponse.newBuilder();
-            done.run(responseBuilder.//
-                    setCompressedRows(ByteString.copyFrom(CompressionUtils.compress(allRows))).//too many array copies 
-                    setStats(Stats.newBuilder().//
-                            setAggregatedRowCount(0).//
-                            setScannedRowCount(0).//
-                            setServiceStartTime(serviceStartTime).//
-                            setServiceEndTime(System.currentTimeMillis()).build()).//
-                    build());
-
-        } catch (IOException ioe) {
-            logger.error(ioe.toString());
-            ResponseConverter.setControllerException(controller, ioe);
-        } finally {
-            IOUtils.closeQuietly(innerScanner);
-            if (region != null) {
-                try {
-                    region.closeRegionOperation();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-    }
-
-    @Override
-    public void start(CoprocessorEnvironment env) throws IOException {
-        if (env instanceof RegionCoprocessorEnvironment) {
-            this.env = (RegionCoprocessorEnvironment) env;
-        } else {
-            throw new CoprocessorException("Must be loaded on a table region!");
-        }
-    }
-
-    @Override
-    public void stop(CoprocessorEnvironment env) throws IOException {
-    }
-
-    @Override
-    public Service getService() {
-        return this;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
deleted file mode 100644
index 40d80ae..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.coprocessor.AggregationCache;
-
-/**
- */
-public class EndpointAggregationCache extends AggregationCache {
-
-    private EndpointAggregators aggregators;
-
-    public EndpointAggregationCache(EndpointAggregators aggregators) {
-        this.aggregators = aggregators;
-    }
-
-    @Override
-    public MeasureAggregator[] createBuffer() {
-        return this.aggregators.createBuffer();
-    }
-
-    public Set<Map.Entry<AggrKey, MeasureAggregator[]>> getAllEntries() {
-        return aggBufMap.entrySet();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
deleted file mode 100644
index 8c3a8bc..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * @author honma
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class EndpointAggregators {
-
-    private enum MetricType {
-        Count, DimensionAsMetric, DistinctCount, Normal
-    }
-
-    private final static class MetricInfo {
-        private MetricType type;
-        private int refIndex = -1;
-        private int precision = -1;
-
-        public MetricInfo(MetricType type, int refIndex, int precision) {
-            this.type = type;
-            this.refIndex = refIndex;
-            this.precision = precision;
-        }
-
-        public MetricInfo(MetricType type, int refIndex) {
-            this.type = type;
-            this.refIndex = refIndex;
-        }
-
-        public MetricInfo(MetricType type) {
-            this.type = type;
-        }
-
-    }
-
-    private static MetricInfo generateMetricInfo(TableRecordInfo tableInfo, FunctionDesc functionDesc) {
-        if (functionDesc.isCount()) {
-            return new MetricInfo(MetricType.Count);
-        } else if (functionDesc.isDimensionAsMetric()) {
-            return new MetricInfo(MetricType.DimensionAsMetric);
-        } else {
-            int index = tableInfo.findFactTableColumn(functionDesc.getParameter().getValue());
-            Preconditions.checkState(index >= 0, "Column " + functionDesc.getParameter().getValue() + " is not found in II");
-            if (functionDesc.isCountDistinct()) {
-                return new MetricInfo(MetricType.DistinctCount, index, functionDesc.getReturnDataType().getPrecision());
-            } else {
-                return new MetricInfo(MetricType.Normal, index);
-            }
-        }
-    }
-
-    public static EndpointAggregators fromFunctions(TableRecordInfo tableInfo, List<FunctionDesc> metrics) {
-        final int metricSize = metrics.size();
-        String[] funcNames = new String[metricSize];
-        String[] dataTypes = new String[metricSize];
-        MetricInfo[] metricInfos = new MetricInfo[metricSize];
-        for (int i = 0; i < metricSize; i++) {
-            FunctionDesc functionDesc = metrics.get(i);
-
-            //TODO: what if funcionDesc's type is different from tablDesc? cause scale difference
-            funcNames[i] = functionDesc.getExpression();
-            dataTypes[i] = functionDesc.getReturnType();
-            metricInfos[i] = generateMetricInfo(tableInfo, functionDesc);
-        }
-
-        return new EndpointAggregators(funcNames, dataTypes, metricInfos, tableInfo.getDigest());
-    }
-
-    final String[] funcNames;
-    final String[] dataTypes;
-    final MetricInfo[] metricInfos;
-
-    final transient TableRecordInfoDigest tableRecordInfoDigest;
-    final transient RawTableRecord rawTableRecord;
-    final transient ImmutableBytesWritable byteBuffer;
-    final transient HyperLogLogPlusCounter[] hllcs;
-    final transient FixedLenMeasureCodec[] measureSerializers;
-    final transient Object[] metricValues;
-
-    final LongMutable ONE = new LongMutable(1);
-
-    private EndpointAggregators(String[] funcNames, String[] dataTypes, MetricInfo[] metricInfos, TableRecordInfoDigest tableInfo) {
-        this.funcNames = funcNames;
-        this.dataTypes = dataTypes;
-        this.metricInfos = metricInfos;
-        this.tableRecordInfoDigest = tableInfo;
-        this.rawTableRecord = tableInfo.createTableRecordBytes();
-        this.byteBuffer = new ImmutableBytesWritable();
-
-        this.hllcs = new HyperLogLogPlusCounter[this.metricInfos.length];
-        this.metricValues = new Object[funcNames.length];
-        this.measureSerializers = new FixedLenMeasureCodec[funcNames.length];
-        for (int i = 0; i < this.measureSerializers.length; ++i) {
-            this.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
-        }
-    }
-
-    public TableRecordInfoDigest getTableRecordInfoDigest() {
-        return tableRecordInfoDigest;
-    }
-
-    public boolean isEmpty() {
-        return !((funcNames != null) && (funcNames.length != 0));
-    }
-
-    public MeasureAggregator[] createBuffer() {
-        MeasureAggregator[] aggrs = new MeasureAggregator[funcNames.length];
-        for (int i = 0; i < aggrs.length; i++) {
-            if (metricInfos[i].type == MetricType.DistinctCount) {
-                aggrs[i] = MeasureAggregator.create(funcNames[i], dataTypes[i]);
-            } else {
-                //all other fixed length measures can be aggregated as long
-                aggrs[i] = MeasureAggregator.create(funcNames[i], "long");
-            }
-        }
-        return aggrs;
-    }
-
-    /**
-     * this method is heavily called at coprocessor side,
-     * Make sure as little object creation as possible
-     */
-    public void aggregate(MeasureAggregator[] measureAggrs, byte[] row) {
-
-        rawTableRecord.setBytes(row, 0, row.length);
-
-        for (int metricIndex = 0; metricIndex < metricInfos.length; ++metricIndex) {
-            final MetricInfo metricInfo = metricInfos[metricIndex];
-            if (metricInfo.type == MetricType.Count) {
-                measureAggrs[metricIndex].aggregate(ONE);
-                continue;
-            }
-
-            if (metricInfo.type == MetricType.DimensionAsMetric) {
-                continue;
-            }
-
-            MeasureAggregator aggregator = measureAggrs[metricIndex];
-            FixedLenMeasureCodec measureSerializer = measureSerializers[metricIndex];
-
-            //get the raw bytes
-            rawTableRecord.getValueBytes(metricInfo.refIndex, byteBuffer);
-
-            if (metricInfo.type == MetricType.Normal) {
-                aggregator.aggregate(measureSerializer.read(byteBuffer.get(), byteBuffer.getOffset()));
-            } else if (metricInfo.type == MetricType.DistinctCount) {
-                //TODO: for unified dictionary, this is okay. but if different data blocks uses different dictionary, we'll have to aggregate original data
-                HyperLogLogPlusCounter hllc = hllcs[metricIndex];
-                if (hllc == null) {
-                    int precision = metricInfo.precision;
-                    hllc = new HyperLogLogPlusCounter(precision);
-                }
-                hllc.clear();
-                hllc.add(byteBuffer.get(), byteBuffer.getOffset(), byteBuffer.getLength());
-                aggregator.aggregate(hllc);
-            }
-        }
-    }
-
-    /**
-     * @param aggrs
-     * @param buffer byte buffer to get the metric data
-     * @return length of metric data
-     */
-    public int serializeMetricValues(MeasureAggregator[] aggrs, byte[] buffer) {
-        for (int i = 0; i < funcNames.length; i++) {
-            metricValues[i] = aggrs[i].getState();
-        }
-
-        int metricBytesOffset = 0;
-        for (int i = 0; i < measureSerializers.length; i++) {
-            measureSerializers[i].write(metricValues[i], buffer, metricBytesOffset);
-            metricBytesOffset += measureSerializers[i].getLength();
-        }
-        return metricBytesOffset;
-    }
-
-    public List<Object> deserializeMetricValues(byte[] metricBytes, int offset) {
-        List<Object> ret = Lists.newArrayList();
-        int metricBytesOffset = offset;
-        for (int i = 0; i < measureSerializers.length; i++) {
-            measureSerializers[i].read(metricBytes, metricBytesOffset);
-            Object valueString = measureSerializers[i].getValue();
-            metricBytesOffset += measureSerializers[i].getLength();
-            ret.add(valueString);
-        }
-        return ret;
-    }
-
-    public static byte[] serialize(EndpointAggregators o) {
-        ByteBuffer buf = ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
-        serializer.serialize(o, buf);
-        byte[] result = new byte[buf.position()];
-        System.arraycopy(buf.array(), 0, result, 0, buf.position());
-        return result;
-    }
-
-    public static EndpointAggregators deserialize(byte[] bytes) {
-        return serializer.deserialize(ByteBuffer.wrap(bytes));
-    }
-
-    private static final Serializer serializer = new Serializer();
-
-    private static class Serializer implements BytesSerializer<EndpointAggregators> {
-
-        @Override
-        public void serialize(EndpointAggregators value, ByteBuffer out) {
-            BytesUtil.writeAsciiStringArray(value.funcNames, out);
-            BytesUtil.writeAsciiStringArray(value.dataTypes, out);
-
-            BytesUtil.writeVInt(value.metricInfos.length, out);
-            for (int i = 0; i < value.metricInfos.length; ++i) {
-                MetricInfo metricInfo = value.metricInfos[i];
-                BytesUtil.writeAsciiString(metricInfo.type.toString(), out);
-                BytesUtil.writeVInt(metricInfo.refIndex, out);
-                BytesUtil.writeVInt(metricInfo.precision, out);
-            }
-
-            BytesUtil.writeByteArray(TableRecordInfoDigest.serialize(value.tableRecordInfoDigest), out);
-        }
-
-        @Override
-        public EndpointAggregators deserialize(ByteBuffer in) {
-
-            String[] funcNames = BytesUtil.readAsciiStringArray(in);
-            String[] dataTypes = BytesUtil.readAsciiStringArray(in);
-
-            int metricInfoLength = BytesUtil.readVInt(in);
-            MetricInfo[] infos = new MetricInfo[metricInfoLength];
-            for (int i = 0; i < infos.length; ++i) {
-                MetricType type = MetricType.valueOf(BytesUtil.readAsciiString(in));
-                int refIndex = BytesUtil.readVInt(in);
-                int presision = BytesUtil.readVInt(in);
-                infos[i] = new MetricInfo(type, refIndex, presision);
-            }
-
-            byte[] temp = BytesUtil.readByteArray(in);
-            TableRecordInfoDigest tableInfo = TableRecordInfoDigest.deserialize(temp);
-
-            return new EndpointAggregators(funcNames, dataTypes, infos, tableInfo);
-        }
-
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java
deleted file mode 100644
index ef033f0..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-public class EndpointEnabler {
-
-    private static final Logger logger = LoggerFactory.getLogger(EndpointEnabler.class);
-
-    static final String FORCE_COPROCESSOR = "forceEndpoint";
-
-    public static boolean isCoprocessorBeneficial() {
-        return Boolean.parseBoolean(getForceCoprocessor());
-    }
-
-    public static void forceCoprocessorOn() {
-        System.setProperty(FORCE_COPROCESSOR, "true");
-    }
-
-    public static void forceCoprocessorOff() {
-        System.setProperty(FORCE_COPROCESSOR, "false");
-    }
-
-    public static String getForceCoprocessor() {
-        return System.getProperty(FORCE_COPROCESSOR);
-    }
-
-    public static void forceCoprocessorUnset() {
-        System.clearProperty(FORCE_COPROCESSOR);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java
deleted file mode 100644
index da1109f..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.util.List;
-
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
-
-public class EndpointTupleConverter {
-
-    final TupleInfo tupleInfo;
-    final List<TblColRef> columns;
-    final int[] columnTupleIdx;
-    final int[] aggrTupleIdx;
-
-    public EndpointTupleConverter(List<TblColRef> columns, List<FunctionDesc> aggrMeasures, TupleInfo returnTupleInfo) {
-        this.tupleInfo = returnTupleInfo;
-        this.columns = columns;
-        this.columnTupleIdx = new int[columns.size()];
-        this.aggrTupleIdx = new int[aggrMeasures.size()];
-
-        for (int i = 0; i < columns.size(); i++) {
-            TblColRef col = columns.get(i);
-            columnTupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
-        }
-
-        for (int i = 0; i < aggrMeasures.size(); i++) {
-            FunctionDesc measure = aggrMeasures.get(i);
-            int tupleIdx;
-            // for dimension playing as metrics, the measure is just a placeholder, the real value comes from columns
-            if (measure.isDimensionAsMetric()) {
-                tupleIdx = -1;
-            }
-            // a rewrite metrics is identified by its rewrite field name
-            else if (measure.needRewrite()) {
-                String rewriteFieldName = measure.getRewriteFieldName();
-                tupleIdx = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
-            }
-            // a non-rewrite metrics (i.e. sum) is like a dimension column
-            else {
-                TblColRef col = measure.getParameter().getColRefs().get(0);
-                tupleIdx = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
-            }
-            aggrTupleIdx[i] = tupleIdx;
-        }
-    }
-
-    public ITuple makeTuple(TableRecord tableRecord, List<Object> measureValues, Tuple tuple) {
-        // dimensions and metrics from II table record 
-        for (int i = 0; i < columnTupleIdx.length; i++) {
-            int tupleIdx = columnTupleIdx[i];
-            if (tupleIdx >= 0) {
-                String value = tableRecord.getValueString(i);
-                tuple.setDimensionValue(tupleIdx, value);
-            }
-        }
-
-        // additional aggregations calculated inside end point (like cube measures)
-        if (measureValues != null) {
-            for (int i = 0; i < aggrTupleIdx.length; ++i) {
-                int tupleIdx = aggrTupleIdx[i];
-                if (tupleIdx >= 0) {
-                    Object value = measureValues.get(i);
-                    if (value instanceof String) {
-                        String dataType = tuple.getDataTypeName(tupleIdx);
-                        value = Tuple.convertOptiqCellValue((String) value, dataType);
-                    }
-                    tuple.setMeasureValue(tupleIdx, value);
-                }
-            }
-        }
-        return tuple;
-    }
-
-}