You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/26 16:08:39 UTC
[16/34] incubator-kylin git commit: KYLIN-960 organize existing
storage module
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
deleted file mode 100644
index 2eba0e3..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.tuple.TupleInfo;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * @author xjiang
- */
-public class SerializedHBaseTupleIterator implements ITupleIterator {
-
- private static final int PARTIAL_DEFAULT_LIMIT = 10000;
-
- private final StorageContext context;
- private final int partialResultLimit;
- private final List<CubeSegmentTupleIterator> segmentIteratorList;
- private final Iterator<CubeSegmentTupleIterator> segmentIteratorIterator;
-
- private ITupleIterator segmentIterator;
- private int scanCount;
- private ITuple next;
-
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
- Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
- StorageContext context, TupleInfo returnTupleInfo) {
-
- this.context = context;
- int limit = context.getLimit();
- this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);
-
- this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
- Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
- for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
- CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
- this.segmentIteratorList.add(segIter);
- }
-
- this.segmentIteratorIterator = this.segmentIteratorList.iterator();
- if (this.segmentIteratorIterator.hasNext()) {
- this.segmentIterator = this.segmentIteratorIterator.next();
- } else {
- this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
- }
- }
-
- private Map<CubeSegment, List<HBaseKeyRange>> makeRangesMap(List<HBaseKeyRange> segmentKeyRanges) {
- Map<CubeSegment, List<HBaseKeyRange>> map = Maps.newHashMap();
- for (HBaseKeyRange range : segmentKeyRanges) {
- List<HBaseKeyRange> list = map.get(range.getCubeSegment());
- if (list == null) {
- list = Lists.newArrayList();
- map.put(range.getCubeSegment(), list);
- }
- list.add(range);
- }
- return map;
- }
-
- @Override
- public boolean hasNext() {
- if (next != null)
- return true;
-
- // 1. check limit
- if (context.isLimitEnabled() && scanCount >= context.getLimit()) {
- return false;
- }
- // 2. check partial result
- if (context.isAcceptPartialResult() && scanCount > partialResultLimit) {
- context.setPartialResultReturned(true);
- return false;
- }
- // 3. check threshold
- if (scanCount >= context.getThreshold()) {
- throw new ScanOutOfLimitException("Scan row count exceeded threshold: " + context.getThreshold() + ", please add filter condition to narrow down backend scan range, like where clause.");
- }
- // 4. check cube segments
- if (segmentIterator.hasNext()) {
- next = segmentIterator.next();
- scanCount++;
- return true;
- } else if (segmentIteratorIterator.hasNext()) {
- segmentIterator.close();
- segmentIterator = segmentIteratorIterator.next();
- return hasNext();
- }
- return false;
- }
-
- @Override
- public ITuple next() {
- if (next == null) {
- hasNext();
- if (next == null)
- throw new NoSuchElementException();
- }
- ITuple r = next;
- next = null;
- return r;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- context.setTotalScanCount(scanCount);
-
- // hasNext() loop may exit because of limit, threshold, etc.
- // close all the remaining segmentIterator
- segmentIterator.close();
- while (segmentIteratorIterator.hasNext()) {
- segmentIterator = segmentIteratorIterator.next();
- segmentIterator.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
deleted file mode 100644
index 9e8c4a0..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggrKey.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Arrays;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.kv.RowConstants;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class AggrKey implements Comparable<AggrKey> {
-
- final byte[] groupByMask;
- final transient int[] groupByMaskSet;
- transient int hashcode;
- byte[] data;
- int offset;
-
- AggrKey(byte[] groupByMask) {
- this.groupByMask = groupByMask;
- LinkedList<Integer> list = Lists.newLinkedList();
- for (int i = 0; i < groupByMask.length; i++) {
- if (groupByMask[i] != 0) {
- list.add(i);
- }
- }
- groupByMaskSet = new int[list.size()];
- for (int i = 0; i < list.size(); i++) {
- groupByMaskSet[i] = list.get(i);
- }
- }
-
- private AggrKey(byte[] groupByMask, int[] groupByMaskSet, byte[] data, int offset, int hashcode) {
- this.groupByMask = groupByMask;
- this.groupByMaskSet = groupByMaskSet;
- this.data = data;
- this.offset = offset;
- this.hashcode = hashcode;
- }
-
- private int calculateHash() {
- int hash = 1;
- for (int i = 0; i < groupByMaskSet.length; i++) {
- byte t = data[offset + groupByMaskSet[i]];
- if (t != RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
- hash = (31 * hash) + t;
- }
- }
- return hash;
- }
-
- public byte[] get() {
- return data;
- }
-
- public int offset() {
- return offset;
- }
-
- public int length() {
- return groupByMask.length;
- }
-
- void set(byte[] data, int offset) {
- this.data = data;
- this.offset = offset;
- this.hashcode = calculateHash();
- }
-
- public byte[] getGroupByMask() {
- return this.groupByMask;
- }
-
- public byte[] copyBytes() {
- return Arrays.copyOfRange(data, offset, offset + length());
- }
-
- AggrKey copy() {
- AggrKey copy = new AggrKey(this.groupByMask, this.groupByMaskSet, copyBytes(), 0, this.hashcode);
- return copy;
- }
-
- @Override
- public int hashCode() {
- return hashcode;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- AggrKey other = (AggrKey) obj;
- return compareTo(other) == 0;
- }
-
- @Override
- public int compareTo(AggrKey o) {
- int comp = this.length() - o.length();
- if (comp != 0)
- return comp;
-
- for (int i = 0; i < groupByMaskSet.length; i++) {
- comp = BytesUtil.compareByteUnsigned(this.data[this.offset + groupByMaskSet[i]], o.data[o.offset + groupByMaskSet[i]]);
- if (comp != 0)
- return comp;
- }
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
deleted file mode 100644
index b7cee9f..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/AggregationCache.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Map;
-
-import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-
-import com.google.common.collect.Maps;
-
-/**
- */
-@SuppressWarnings("rawtypes")
-public abstract class AggregationCache {
- static final long MEMORY_USAGE_CAP = 500 * 1024 * 1024; // 500 MB
- static final long MEMOERY_MAX_BYTES = Runtime.getRuntime().maxMemory();
- protected final Map<AggrKey, MeasureAggregator[]> aggBufMap;
- transient int rowMemBytes;
- private AggrKey firstKey = null;
-
- public AggregationCache() {
- this.aggBufMap = Maps.newHashMap();
- }
-
- public abstract MeasureAggregator[] createBuffer();
-
- public MeasureAggregator[] getBuffer(AggrKey aggkey) {
- MeasureAggregator[] aggBuf = aggBufMap.get(aggkey);
- if (aggBuf == null) {
- aggBuf = createBuffer();
- AggrKey key = aggkey.copy();
- aggBufMap.put(key, aggBuf);
-
- if (firstKey == null) {
- firstKey = key;
- }
- }
- return aggBuf;
- }
-
- public long getSize() {
- return aggBufMap.size();
- }
-
- public void checkMemoryUsage() {
- if (firstKey == null)
- return;
-
- // about memory calculation,
- // http://seniorjava.wordpress.com/2013/09/01/java-objects-memory-size-reference/
- if (rowMemBytes <= 0) {
- if (aggBufMap.size() > 0) {
- rowMemBytes = 0;
- MeasureAggregator[] measureAggregators = aggBufMap.get(firstKey);
- for (MeasureAggregator agg : measureAggregators) {
- rowMemBytes += agg.getMemBytesEstimate();
- }
- }
- }
- int size = aggBufMap.size();
- long memUsage = (40 + rowMemBytes) * size;
- if (memUsage > MEMORY_USAGE_CAP) {
- throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor.");
- }
-
- //If less than 5% of max memory
- long avail = MemoryBudgetController.getSystemAvailBytes();
- if (avail < (MEMOERY_MAX_BYTES / 20)) {
- throw new RuntimeException("Running Kylin coprocessor when too little memory is left. Abort coprocessor. Current available memory is " + avail + ". Max memory is " + MEMOERY_MAX_BYTES);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
deleted file mode 100644
index 6e69827..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorConstants.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-/**
- */
-public class CoprocessorConstants {
- public static final int SERIALIZE_BUFFER_SIZE = 65536;
- public static final int METRIC_SERIALIZE_BUFFER_SIZE = 65536;
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
deleted file mode 100644
index feb36ca..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorFilter.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Set;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-/**
- * @author yangli9
- */
-public class CoprocessorFilter {
-
- public static CoprocessorFilter fromFilter(final IDictionaryAware dictionaryAware, TupleFilter rootFilter, FilterDecorator.FilterConstantsTreatment filterConstantsTreatment) {
- // translate constants into dictionary IDs via a serialize copy
- FilterDecorator filterDecorator = new FilterDecorator(dictionaryAware, filterConstantsTreatment);
- byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator, DictCodeSystem.INSTANCE);
- TupleFilter copy = TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
-
- return new CoprocessorFilter(copy, filterDecorator.getInevaluableColumns());
- }
-
- public static byte[] serialize(CoprocessorFilter o) {
- return (o.filter == null) ? BytesUtil.EMPTY_BYTE_ARRAY : TupleFilterSerializer.serialize(o.filter, DictCodeSystem.INSTANCE);
- }
-
- public static CoprocessorFilter deserialize(byte[] filterBytes) {
- TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null //
- : TupleFilterSerializer.deserialize(filterBytes, DictCodeSystem.INSTANCE);
- return new CoprocessorFilter(filter, null);
- }
-
- // ============================================================================
-
- private final TupleFilter filter;
- private final Set<TblColRef> inevaluableColumns;
-
- public CoprocessorFilter(TupleFilter filter, Set<TblColRef> inevaluableColumns) {
- this.filter = filter;
- this.inevaluableColumns = inevaluableColumns;
- }
-
- public TupleFilter getFilter() {
- return filter;
- }
-
- public Set<TblColRef> getInevaluableColumns() {
- return inevaluableColumns;
- }
-
- public boolean evaluate(IEvaluatableTuple tuple) {
- if (filter == null)
- return true;
- else
- return filter.evaluate(tuple, DictCodeSystem.INSTANCE);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
deleted file mode 100644
index 2821a6c..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorProjector.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowKeyEncoder;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author yangli9
- */
-public class CoprocessorProjector {
-
- public static CoprocessorProjector makeForObserver(final CubeSegment cubeSegment, final Cuboid cuboid, final Collection<TblColRef> dimensionColumns) {
-
- RowKeyEncoder rowKeyMaskEncoder = new RowKeyEncoder(cubeSegment, cuboid) {
- @Override
- protected int fillHeader(byte[] bytes, byte[][] values) {
- Arrays.fill(bytes, 0, this.headerLength, (byte) 0xff);
- return this.headerLength;
- }
-
- @Override
- protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) {
- byte bits = dimensionColumns.contains(column) ? (byte) 0xff : 0x00;
- Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, bits);
- }
- };
-
- byte[] mask = rowKeyMaskEncoder.encode(new byte[cuboid.getColumns().size()][]);
- return new CoprocessorProjector(mask, dimensionColumns.size() != 0);
- }
-
- public static CoprocessorProjector makeForEndpoint(final TableRecordInfo tableInfo, final Collection<TblColRef> groupby) {
- byte[] mask = new byte[tableInfo.getDigest().getByteFormLen()];
- int maskIdx = 0;
- for (int i = 0; i < tableInfo.getDigest().getColumnCount(); ++i) {
- TblColRef tblColRef = tableInfo.getColumns().get(i);
- int length = tableInfo.getDigest().length(i);
- byte bits = groupby.contains(tblColRef) ? (byte) 0xff : 0x00;
- for (int j = 0; j < length; ++j) {
- mask[maskIdx++] = bits;
- }
- }
- return new CoprocessorProjector(mask, groupby.size() != 0);
- }
-
- public static byte[] serialize(CoprocessorProjector o) {
- ByteBuffer buf = ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static CoprocessorProjector deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- private static final Serializer serializer = new Serializer();
-
- private static class Serializer implements BytesSerializer<CoprocessorProjector> {
-
- @Override
- public void serialize(CoprocessorProjector value, ByteBuffer out) {
- BytesUtil.writeByteArray(value.groupByMask, out);
- BytesUtil.writeVInt(value.hasGroupby ? 1 : 0, out);
- }
-
- @Override
- public CoprocessorProjector deserialize(ByteBuffer in) {
- byte[] mask = BytesUtil.readByteArray(in);
- boolean hasGroupBy = BytesUtil.readVInt(in) == 1;
- return new CoprocessorProjector(mask, hasGroupBy);
- }
- }
-
- // ============================================================================
-
- final transient AggrKey aggrKey;
- final byte[] groupByMask; // mask out columns that are not needed (by group by)
- final boolean hasGroupby;
-
- public CoprocessorProjector(byte[] groupByMask, boolean hasGroupby) {
- this.groupByMask = groupByMask;
- this.aggrKey = new AggrKey(groupByMask);
- this.hasGroupby = hasGroupby;
- }
-
- public boolean hasGroupby() {
- return hasGroupby;
- }
-
- public AggrKey getAggrKey(List<Cell> rowCells) {
- Cell cell = rowCells.get(0);
- assert groupByMask.length == cell.getRowLength();
-
- aggrKey.set(cell.getRowArray(), cell.getRowOffset());
- return aggrKey;
- }
-
- public AggrKey getAggrKey(byte[] row) {
- assert groupByMask.length == row.length;
- aggrKey.set(row, 0);
- return aggrKey;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java
deleted file mode 100644
index 11cb4e8..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/CoprocessorRowType.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- */
-public class CoprocessorRowType {
-
- //for endpoint
- public static CoprocessorRowType fromTableRecordInfo(TableRecordInfo tableRecordInfo, List<TblColRef> cols) {
-
- int[] colSizes = new int[cols.size()];
- for (int i = 0; i < cols.size(); i++) {
- colSizes[i] = tableRecordInfo.getDigest().length(i);
- }
- return new CoprocessorRowType(cols.toArray(new TblColRef[cols.size()]), colSizes);
- }
-
- //for observer
- public static CoprocessorRowType fromCuboid(CubeSegment seg, Cuboid cuboid) {
- List<TblColRef> colList = cuboid.getColumns();
- TblColRef[] cols = colList.toArray(new TblColRef[colList.size()]);
- RowKeyColumnIO colIO = new RowKeyColumnIO(seg);
- int[] colSizes = new int[cols.length];
- for (int i = 0; i < cols.length; i++) {
- colSizes[i] = colIO.getColumnLength(cols[i]);
- }
- return new CoprocessorRowType(cols, colSizes);
- }
-
- public static byte[] serialize(CoprocessorRowType o) {
- ByteBuffer buf = ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static CoprocessorRowType deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- private static final Serializer serializer = new Serializer();
-
- private static class Serializer implements BytesSerializer<CoprocessorRowType> {
-
- @Override
- public void serialize(CoprocessorRowType o, ByteBuffer out) {
- int n = o.columns.length;
- BytesUtil.writeVInt(o.columns.length, out);
- for (int i = 0; i < n; i++) {
- BytesUtil.writeAsciiString(o.columns[i].getTable(), out);
- BytesUtil.writeAsciiString(o.columns[i].getName(), out);
- BytesUtil.writeVInt(o.columnSizes[i], out);
- }
- }
-
- @Override
- public CoprocessorRowType deserialize(ByteBuffer in) {
- int n = BytesUtil.readVInt(in);
- TblColRef[] cols = new TblColRef[n];
- int[] colSizes = new int[n];
- for (int i = 0; i < n; i++) {
- String tableName = BytesUtil.readAsciiString(in);
- String colName = BytesUtil.readAsciiString(in);
- TableDesc table = new TableDesc();
- table.setName(tableName);
- ColumnDesc col = new ColumnDesc();
- col.setTable(table);
- col.setName(colName);
- col.init(table);
- cols[i] = new TblColRef(col);
-
- int colSize = BytesUtil.readVInt(in);
- colSizes[i] = colSize;
- }
- return new CoprocessorRowType(cols, colSizes);
- }
- }
-
- // ============================================================================
-
- public TblColRef[] columns;
- public int[] columnSizes;
- public int[] columnOffsets;
- public HashMap<TblColRef, Integer> columnIdxMap;
-
- public CoprocessorRowType(TblColRef[] columns, int[] columnSizes) {
- this.columns = columns;
- this.columnSizes = columnSizes;
- init();
- }
-
- public int getColIndexByTblColRef(TblColRef colRef) {
- return columnIdxMap.get(colRef);
- }
-
- private void init() {
- int[] offsets = new int[columns.length];
- int o = RowConstants.ROWKEY_CUBOIDID_LEN;
- for (int i = 0; i < columns.length; i++) {
- offsets[i] = o;
- o += columnSizes[i];
- }
- this.columnOffsets = offsets;
-
- HashMap<TblColRef, Integer> map = Maps.newHashMap();
- for (int i = 0; i < columns.length; i++) {
- map.put(columns[i], i);
- }
- this.columnIdxMap = map;
- }
-
- public int getColumnCount() {
- return columns.length;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
deleted file mode 100644
index d4e2713..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/DictCodeSystem.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.filter.IFilterCodeSystem;
-
-/**
- * A simple code system where all values are dictionary IDs (fixed length bytes) encoded as ISO-8859-1 strings.
- *
- * @author yangli9
- */
-public class DictCodeSystem implements IFilterCodeSystem<String> {
-
- public static final DictCodeSystem INSTANCE = new DictCodeSystem();
-
- private DictCodeSystem() {
- // singleton
- }
-
- @Override
- public boolean isNull(String value) {
- if (value == null)
- return true;
-
- String v = value;
- for (int i = 0, n = v.length(); i < n; i++) {
- if ((byte) v.charAt(i) != Dictionary.NULL)
- return false;
- }
- return true;
- }
-
- @Override
- public int compare(String tupleValue, String constValue) {
- return tupleValue.compareTo(constValue);
- }
-
- //TODO: should use ISO-8859-1 rather than UTF8
- @Override
- public void serialize(String value, ByteBuffer buffer) {
- BytesUtil.writeUTFString(value, buffer);
- }
-
- @Override
- public String deserialize(ByteBuffer buffer) {
- return BytesUtil.readUTFString(buffer);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
deleted file mode 100644
index 350b167..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/FilterDecorator.java
+++ /dev/null
@@ -1,183 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor;
-
-import java.util.Collection;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Sets;
-
-/**
- */
-@SuppressWarnings("unchecked")
-public class FilterDecorator implements TupleFilterSerializer.Decorator {
- public enum FilterConstantsTreatment {
- AS_IT_IS, REPLACE_WITH_GLOBAL_DICT, REPLACE_WITH_LOCAL_DICT
- }
-
- private RowKeyColumnIO columnIO;
- private Set<TblColRef> inevaluableColumns;
- private FilterConstantsTreatment filterConstantsTreatment;
-
- public FilterDecorator(IDictionaryAware seg, FilterConstantsTreatment filterConstantsTreatment) {
- this.columnIO = new RowKeyColumnIO(seg);
- this.inevaluableColumns = Sets.newHashSet();
- this.filterConstantsTreatment = filterConstantsTreatment;
- }
-
- public Set<TblColRef> getInevaluableColumns() {
- return inevaluableColumns;
- }
-
- private TupleFilter replaceConstantsWithLocalDict(CompareTupleFilter oldCompareFilter, CompareTupleFilter newCompareFilter) {
- //TODO localdict: (performance issue) transalte() with roundingflag 0 will use try catch exceptions to deal with non-existing entries
- return replaceConstantsWithGlobalDict(oldCompareFilter, newCompareFilter);
- }
-
- private TupleFilter replaceConstantsWithGlobalDict(CompareTupleFilter oldCompareFilter, CompareTupleFilter newCompareFilter) {
- Collection<String> constValues = (Collection<String>) oldCompareFilter.getValues();
- String firstValue = constValues.iterator().next();
- TblColRef col = newCompareFilter.getColumn();
-
- TupleFilter result;
- String v;
-
- // translate constant into rowkey ID
- switch (newCompareFilter.getOperator()) {
- case EQ:
- case IN:
- Set<String> newValues = Sets.newHashSet();
- for (String value : constValues) {
- v = translate(col, value, 0);
- if (!isDictNull(v))
- newValues.add(v);
- }
- if (newValues.isEmpty()) {
- result = ConstantTupleFilter.FALSE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(newValues));
- result = newCompareFilter;
- }
- break;
- case NEQ:
- v = translate(col, firstValue, 0);
- if (isDictNull(v)) {
- result = ConstantTupleFilter.TRUE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(v));
- result = newCompareFilter;
- }
- break;
- case LT:
- v = translate(col, firstValue, 1);
- if (isDictNull(v)) {
- result = ConstantTupleFilter.TRUE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(v));
- result = newCompareFilter;
- }
- break;
- case LTE:
- v = translate(col, firstValue, -1);
- if (isDictNull(v)) {
- result = ConstantTupleFilter.FALSE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(v));
- result = newCompareFilter;
- }
- break;
- case GT:
- v = translate(col, firstValue, -1);
- if (isDictNull(v)) {
- result = ConstantTupleFilter.TRUE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(v));
- result = newCompareFilter;
- }
- break;
- case GTE:
- v = translate(col, firstValue, 1);
- if (isDictNull(v)) {
- result = ConstantTupleFilter.FALSE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(v));
- result = newCompareFilter;
- }
- break;
- default:
- throw new IllegalStateException("Cannot handle operator " + newCompareFilter.getOperator());
- }
- return result;
- }
-
- private boolean isDictNull(String v) {
- return DictCodeSystem.INSTANCE.isNull(v);
- }
-
- @Override
- public TupleFilter onSerialize(TupleFilter filter) {
- if (filter == null)
- return null;
-
- // In case of NOT(inEvaluatableFilter), we should immediately replace it as TRUE,
- // Otherwise, inEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
- // always return FALSE
- if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
- TupleFilter.collectColumns(filter, inevaluableColumns);
- return ConstantTupleFilter.TRUE;
- }
-
- if (!(filter instanceof CompareTupleFilter))
- return filter;
-
- if (!TupleFilter.isEvaluableRecursively(filter)) {
- TupleFilter.collectColumns(filter, inevaluableColumns);
- return ConstantTupleFilter.TRUE;
- }
-
- if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
- return filter;
- } else {
-
- // extract ColumnFilter & ConstantFilter
- CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
- TblColRef col = compareFilter.getColumn();
-
- if (col == null) {
- return filter;
- }
-
- Collection<String> constValues = (Collection<String>) compareFilter.getValues();
- if (constValues == null || constValues.isEmpty()) {
- return filter;
- }
-
- CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
- newCompareFilter.addChild(new ColumnTupleFilter(col));
-
- if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
- return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
- } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
- return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
- } else {
- throw new RuntimeException("should not reach here");
- }
- }
- }
-
- private String translate(TblColRef column, String v, int roundingFlag) {
- byte[] value = Bytes.toBytes(v);
- byte[] id = new byte[columnIO.getColumnLength(column)];
- columnIO.writeColumn(column, value, value.length, roundingFlag, Dictionary.NULL, id, 0);
- return Dictionary.dictIdToString(id, 0, id.length);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
deleted file mode 100644
index f2c19d9..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/ClearTextDictionary.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.util.Map;
-
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
-
-import com.google.common.collect.Maps;
-
-/**
- */
-public class ClearTextDictionary implements IDictionaryAware {
-
- private final TableRecordInfoDigest digest;
- private final Map<TblColRef, Integer> columnIndexMap;
-
- public ClearTextDictionary(TableRecordInfoDigest digest, CoprocessorRowType coprocessorRowType) {
- this.digest = digest;
- this.columnIndexMap = coprocessorRowType.columnIdxMap;
- }
-
- public ClearTextDictionary(TableRecordInfo tableRecordInfo) {
- this.digest = tableRecordInfo.getDigest();
- this.columnIndexMap = Maps.newHashMap();
- for (int i = 0; i < tableRecordInfo.getColumns().size(); i++) {
- columnIndexMap.put(tableRecordInfo.getColumns().get(i), i);
- }
- }
-
- @Override
- public int getColumnLength(TblColRef col) {
- return digest.length(columnIndexMap.get(col));
- }
-
- @Override
- public Dictionary<?> getDictionary(TblColRef col) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
deleted file mode 100644
index c44ac52..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.protobuf.ResponseConverter;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.CompressionUtils;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.util.KryoUtils;
-import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.gridtable.GTScanRequest;
-import org.apache.kylin.gridtable.IGTScanner;
-import org.apache.kylin.gridtable.IGTStore;
-import org.apache.kylin.storage.cube.CellListIterator;
-import org.apache.kylin.storage.cube.CubeHBaseRPC;
-import org.apache.kylin.storage.cube.HBaseReadonlyStore;
-import org.apache.kylin.storage.cube.RawScan;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.CubeVisitProtos;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Builder;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.Service;
-
-public class CubeVisitService extends CubeVisitProtos.CubeVisitService implements Coprocessor, CoprocessorService {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeVisitService.class);
- //TODO limit memory footprint
- private static final int MEMORY_LIMIT = 500 * 1024 * 1024;
-
- private RegionCoprocessorEnvironment env;
-
- private long serviceStartTime;
-
- static class InnerScannerAsIterator implements CellListIterator {
- private RegionScanner regionScanner;
- private List<Cell> nextOne = Lists.newArrayList();
- private List<Cell> ret = Lists.newArrayList();
-
- private boolean hasMore;
-
- public InnerScannerAsIterator(RegionScanner regionScanner) {
- this.regionScanner = regionScanner;
-
- try {
- hasMore = regionScanner.nextRaw(nextOne);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public boolean hasNext() {
- return !nextOne.isEmpty();
- }
-
- @Override
- public List<Cell> next() {
-
- if (nextOne.size() < 1) {
- throw new IllegalStateException();
- }
- ret.clear();
- ret.addAll(nextOne);
- nextOne.clear();
- try {
- if (hasMore) {
- hasMore = regionScanner.nextRaw(nextOne);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return ret;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() throws IOException {
- //does not need to close as regionScanner will be closed in finally block
- }
- }
-
- @Override
- public void visitCube(RpcController controller, CubeVisitProtos.CubeVisitRequest request, RpcCallback<CubeVisitProtos.CubeVisitResponse> done) {
- RegionScanner innerScanner = null;
- HRegion region = null;
-
- try {
- this.serviceStartTime = System.currentTimeMillis();
-
- GTScanRequest scanReq = KryoUtils.deserialize(request.getGtScanRequest().toByteArray(), GTScanRequest.class);
- RawScan hbaseRawScan = KryoUtils.deserialize(request.getHbaseRawScan().toByteArray(), RawScan.class);
- //TODO: rewrite own start/end
- Scan scan = CubeHBaseRPC.buildScan(hbaseRawScan);
-
- region = env.getRegion();
- region.startRegionOperation();
-
- innerScanner = region.getScanner(scan);
- InnerScannerAsIterator cellListIterator = new InnerScannerAsIterator(innerScanner);
-
- IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScan.hbaseColumns);
- IGTScanner rawScanner = store.scan(scanReq);
- IGTScanner finalScanner = scanReq.decorateScanner(rawScanner);
-
- ByteBuffer buffer = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream(RowConstants.ROWVALUE_BUFFER_SIZE);
- for (GTRecord oneRecord : finalScanner) {
- buffer.clear();
- oneRecord.exportAllColumns(buffer);
- buffer.flip();
- outputStream.write(buffer.array(), buffer.arrayOffset() - buffer.position(), buffer.remaining());
- }
- //outputStream.close() is not necessary
- byte[] allRows = outputStream.toByteArray();
- Builder responseBuilder = CubeVisitProtos.CubeVisitResponse.newBuilder();
- done.run(responseBuilder.//
- setCompressedRows(ByteString.copyFrom(CompressionUtils.compress(allRows))).//too many array copies
- setStats(Stats.newBuilder().//
- setAggregatedRowCount(0).//
- setScannedRowCount(0).//
- setServiceStartTime(serviceStartTime).//
- setServiceEndTime(System.currentTimeMillis()).build()).//
- build());
-
- } catch (IOException ioe) {
- logger.error(ioe.toString());
- ResponseConverter.setControllerException(controller, ioe);
- } finally {
- IOUtils.closeQuietly(innerScanner);
- if (region != null) {
- try {
- region.closeRegionOperation();
- } catch (IOException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- @Override
- public void start(CoprocessorEnvironment env) throws IOException {
- if (env instanceof RegionCoprocessorEnvironment) {
- this.env = (RegionCoprocessorEnvironment) env;
- } else {
- throw new CoprocessorException("Must be loaded on a table region!");
- }
- }
-
- @Override
- public void stop(CoprocessorEnvironment env) throws IOException {
- }
-
- @Override
- public Service getService() {
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
deleted file mode 100644
index 40d80ae..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregationCache.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.coprocessor.AggregationCache;
-
-/**
- */
-public class EndpointAggregationCache extends AggregationCache {
-
- private EndpointAggregators aggregators;
-
- public EndpointAggregationCache(EndpointAggregators aggregators) {
- this.aggregators = aggregators;
- }
-
- @Override
- public MeasureAggregator[] createBuffer() {
- return this.aggregators.createBuffer();
- }
-
- public Set<Map.Entry<AggrKey, MeasureAggregator[]>> getAllEntries() {
- return aggBufMap.entrySet();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
deleted file mode 100644
index 8c3a8bc..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * @author honma
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class EndpointAggregators {
-
- private enum MetricType {
- Count, DimensionAsMetric, DistinctCount, Normal
- }
-
- private final static class MetricInfo {
- private MetricType type;
- private int refIndex = -1;
- private int precision = -1;
-
- public MetricInfo(MetricType type, int refIndex, int precision) {
- this.type = type;
- this.refIndex = refIndex;
- this.precision = precision;
- }
-
- public MetricInfo(MetricType type, int refIndex) {
- this.type = type;
- this.refIndex = refIndex;
- }
-
- public MetricInfo(MetricType type) {
- this.type = type;
- }
-
- }
-
- private static MetricInfo generateMetricInfo(TableRecordInfo tableInfo, FunctionDesc functionDesc) {
- if (functionDesc.isCount()) {
- return new MetricInfo(MetricType.Count);
- } else if (functionDesc.isDimensionAsMetric()) {
- return new MetricInfo(MetricType.DimensionAsMetric);
- } else {
- int index = tableInfo.findFactTableColumn(functionDesc.getParameter().getValue());
- Preconditions.checkState(index >= 0, "Column " + functionDesc.getParameter().getValue() + " is not found in II");
- if (functionDesc.isCountDistinct()) {
- return new MetricInfo(MetricType.DistinctCount, index, functionDesc.getReturnDataType().getPrecision());
- } else {
- return new MetricInfo(MetricType.Normal, index);
- }
- }
- }
-
- public static EndpointAggregators fromFunctions(TableRecordInfo tableInfo, List<FunctionDesc> metrics) {
- final int metricSize = metrics.size();
- String[] funcNames = new String[metricSize];
- String[] dataTypes = new String[metricSize];
- MetricInfo[] metricInfos = new MetricInfo[metricSize];
- for (int i = 0; i < metricSize; i++) {
- FunctionDesc functionDesc = metrics.get(i);
-
- //TODO: what if funcionDesc's type is different from tablDesc? cause scale difference
- funcNames[i] = functionDesc.getExpression();
- dataTypes[i] = functionDesc.getReturnType();
- metricInfos[i] = generateMetricInfo(tableInfo, functionDesc);
- }
-
- return new EndpointAggregators(funcNames, dataTypes, metricInfos, tableInfo.getDigest());
- }
-
- final String[] funcNames;
- final String[] dataTypes;
- final MetricInfo[] metricInfos;
-
- final transient TableRecordInfoDigest tableRecordInfoDigest;
- final transient RawTableRecord rawTableRecord;
- final transient ImmutableBytesWritable byteBuffer;
- final transient HyperLogLogPlusCounter[] hllcs;
- final transient FixedLenMeasureCodec[] measureSerializers;
- final transient Object[] metricValues;
-
- final LongMutable ONE = new LongMutable(1);
-
- private EndpointAggregators(String[] funcNames, String[] dataTypes, MetricInfo[] metricInfos, TableRecordInfoDigest tableInfo) {
- this.funcNames = funcNames;
- this.dataTypes = dataTypes;
- this.metricInfos = metricInfos;
- this.tableRecordInfoDigest = tableInfo;
- this.rawTableRecord = tableInfo.createTableRecordBytes();
- this.byteBuffer = new ImmutableBytesWritable();
-
- this.hllcs = new HyperLogLogPlusCounter[this.metricInfos.length];
- this.metricValues = new Object[funcNames.length];
- this.measureSerializers = new FixedLenMeasureCodec[funcNames.length];
- for (int i = 0; i < this.measureSerializers.length; ++i) {
- this.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
- }
- }
-
- public TableRecordInfoDigest getTableRecordInfoDigest() {
- return tableRecordInfoDigest;
- }
-
- public boolean isEmpty() {
- return !((funcNames != null) && (funcNames.length != 0));
- }
-
- public MeasureAggregator[] createBuffer() {
- MeasureAggregator[] aggrs = new MeasureAggregator[funcNames.length];
- for (int i = 0; i < aggrs.length; i++) {
- if (metricInfos[i].type == MetricType.DistinctCount) {
- aggrs[i] = MeasureAggregator.create(funcNames[i], dataTypes[i]);
- } else {
- //all other fixed length measures can be aggregated as long
- aggrs[i] = MeasureAggregator.create(funcNames[i], "long");
- }
- }
- return aggrs;
- }
-
- /**
- * this method is heavily called at coprocessor side,
- * Make sure as little object creation as possible
- */
- public void aggregate(MeasureAggregator[] measureAggrs, byte[] row) {
-
- rawTableRecord.setBytes(row, 0, row.length);
-
- for (int metricIndex = 0; metricIndex < metricInfos.length; ++metricIndex) {
- final MetricInfo metricInfo = metricInfos[metricIndex];
- if (metricInfo.type == MetricType.Count) {
- measureAggrs[metricIndex].aggregate(ONE);
- continue;
- }
-
- if (metricInfo.type == MetricType.DimensionAsMetric) {
- continue;
- }
-
- MeasureAggregator aggregator = measureAggrs[metricIndex];
- FixedLenMeasureCodec measureSerializer = measureSerializers[metricIndex];
-
- //get the raw bytes
- rawTableRecord.getValueBytes(metricInfo.refIndex, byteBuffer);
-
- if (metricInfo.type == MetricType.Normal) {
- aggregator.aggregate(measureSerializer.read(byteBuffer.get(), byteBuffer.getOffset()));
- } else if (metricInfo.type == MetricType.DistinctCount) {
- //TODO: for unified dictionary, this is okay. but if different data blocks uses different dictionary, we'll have to aggregate original data
- HyperLogLogPlusCounter hllc = hllcs[metricIndex];
- if (hllc == null) {
- int precision = metricInfo.precision;
- hllc = new HyperLogLogPlusCounter(precision);
- }
- hllc.clear();
- hllc.add(byteBuffer.get(), byteBuffer.getOffset(), byteBuffer.getLength());
- aggregator.aggregate(hllc);
- }
- }
- }
-
- /**
- * @param aggrs
- * @param buffer byte buffer to get the metric data
- * @return length of metric data
- */
- public int serializeMetricValues(MeasureAggregator[] aggrs, byte[] buffer) {
- for (int i = 0; i < funcNames.length; i++) {
- metricValues[i] = aggrs[i].getState();
- }
-
- int metricBytesOffset = 0;
- for (int i = 0; i < measureSerializers.length; i++) {
- measureSerializers[i].write(metricValues[i], buffer, metricBytesOffset);
- metricBytesOffset += measureSerializers[i].getLength();
- }
- return metricBytesOffset;
- }
-
- public List<Object> deserializeMetricValues(byte[] metricBytes, int offset) {
- List<Object> ret = Lists.newArrayList();
- int metricBytesOffset = offset;
- for (int i = 0; i < measureSerializers.length; i++) {
- measureSerializers[i].read(metricBytes, metricBytesOffset);
- Object valueString = measureSerializers[i].getValue();
- metricBytesOffset += measureSerializers[i].getLength();
- ret.add(valueString);
- }
- return ret;
- }
-
- public static byte[] serialize(EndpointAggregators o) {
- ByteBuffer buf = ByteBuffer.allocate(CoprocessorConstants.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static EndpointAggregators deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- private static final Serializer serializer = new Serializer();
-
- private static class Serializer implements BytesSerializer<EndpointAggregators> {
-
- @Override
- public void serialize(EndpointAggregators value, ByteBuffer out) {
- BytesUtil.writeAsciiStringArray(value.funcNames, out);
- BytesUtil.writeAsciiStringArray(value.dataTypes, out);
-
- BytesUtil.writeVInt(value.metricInfos.length, out);
- for (int i = 0; i < value.metricInfos.length; ++i) {
- MetricInfo metricInfo = value.metricInfos[i];
- BytesUtil.writeAsciiString(metricInfo.type.toString(), out);
- BytesUtil.writeVInt(metricInfo.refIndex, out);
- BytesUtil.writeVInt(metricInfo.precision, out);
- }
-
- BytesUtil.writeByteArray(TableRecordInfoDigest.serialize(value.tableRecordInfoDigest), out);
- }
-
- @Override
- public EndpointAggregators deserialize(ByteBuffer in) {
-
- String[] funcNames = BytesUtil.readAsciiStringArray(in);
- String[] dataTypes = BytesUtil.readAsciiStringArray(in);
-
- int metricInfoLength = BytesUtil.readVInt(in);
- MetricInfo[] infos = new MetricInfo[metricInfoLength];
- for (int i = 0; i < infos.length; ++i) {
- MetricType type = MetricType.valueOf(BytesUtil.readAsciiString(in));
- int refIndex = BytesUtil.readVInt(in);
- int presision = BytesUtil.readVInt(in);
- infos[i] = new MetricInfo(type, refIndex, presision);
- }
-
- byte[] temp = BytesUtil.readByteArray(in);
- TableRecordInfoDigest tableInfo = TableRecordInfoDigest.deserialize(temp);
-
- return new EndpointAggregators(funcNames, dataTypes, infos, tableInfo);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java
deleted file mode 100644
index ef033f0..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointEnabler.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-public class EndpointEnabler {
-
- private static final Logger logger = LoggerFactory.getLogger(EndpointEnabler.class);
-
- static final String FORCE_COPROCESSOR = "forceEndpoint";
-
- public static boolean isCoprocessorBeneficial() {
- return Boolean.parseBoolean(getForceCoprocessor());
- }
-
- public static void forceCoprocessorOn() {
- System.setProperty(FORCE_COPROCESSOR, "true");
- }
-
- public static void forceCoprocessorOff() {
- System.setProperty(FORCE_COPROCESSOR, "false");
- }
-
- public static String getForceCoprocessor() {
- return System.getProperty(FORCE_COPROCESSOR);
- }
-
- public static void forceCoprocessorUnset() {
- System.clearProperty(FORCE_COPROCESSOR);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java
deleted file mode 100644
index da1109f..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleConverter.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-
-import java.util.List;
-
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
-
-public class EndpointTupleConverter {
-
- final TupleInfo tupleInfo;
- final List<TblColRef> columns;
- final int[] columnTupleIdx;
- final int[] aggrTupleIdx;
-
- public EndpointTupleConverter(List<TblColRef> columns, List<FunctionDesc> aggrMeasures, TupleInfo returnTupleInfo) {
- this.tupleInfo = returnTupleInfo;
- this.columns = columns;
- this.columnTupleIdx = new int[columns.size()];
- this.aggrTupleIdx = new int[aggrMeasures.size()];
-
- for (int i = 0; i < columns.size(); i++) {
- TblColRef col = columns.get(i);
- columnTupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
- }
-
- for (int i = 0; i < aggrMeasures.size(); i++) {
- FunctionDesc measure = aggrMeasures.get(i);
- int tupleIdx;
- // for dimension playing as metrics, the measure is just a placeholder, the real value comes from columns
- if (measure.isDimensionAsMetric()) {
- tupleIdx = -1;
- }
- // a rewrite metrics is identified by its rewrite field name
- else if (measure.needRewrite()) {
- String rewriteFieldName = measure.getRewriteFieldName();
- tupleIdx = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
- }
- // a non-rewrite metrics (i.e. sum) is like a dimension column
- else {
- TblColRef col = measure.getParameter().getColRefs().get(0);
- tupleIdx = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
- }
- aggrTupleIdx[i] = tupleIdx;
- }
- }
-
- public ITuple makeTuple(TableRecord tableRecord, List<Object> measureValues, Tuple tuple) {
- // dimensions and metrics from II table record
- for (int i = 0; i < columnTupleIdx.length; i++) {
- int tupleIdx = columnTupleIdx[i];
- if (tupleIdx >= 0) {
- String value = tableRecord.getValueString(i);
- tuple.setDimensionValue(tupleIdx, value);
- }
- }
-
- // additional aggregations calculated inside end point (like cube measures)
- if (measureValues != null) {
- for (int i = 0; i < aggrTupleIdx.length; ++i) {
- int tupleIdx = aggrTupleIdx[i];
- if (tupleIdx >= 0) {
- Object value = measureValues.get(i);
- if (value instanceof String) {
- String dataType = tuple.getDataTypeName(tupleIdx);
- value = Tuple.convertOptiqCellValue((String) value, dataType);
- }
- tuple.setMeasureValue(tupleIdx, value);
- }
- }
- }
- return tuple;
- }
-
-}