You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/07/08 07:15:19 UTC
[6/8] kylin git commit: KYLIN-1858 remove all ii related code
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
deleted file mode 100644
index 0ee16b8..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-//import it.uniroma3.mat.extendedset.intset.ConciseSet;
-//
-//import java.nio.ByteBuffer;
-//import java.nio.IntBuffer;
-//import java.util.ArrayList;
-//import java.util.Arrays;
-//import java.util.List;
-//
-//import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-//
-//import org.apache.kylin.common.util.BytesUtil;
-//import org.apache.kylin.dict.Dictionary;
-
-/**
- * @author yangli9
- */
-//public class BitMapContainer implements ColumnValueContainer {
-//
-// int valueLen;
-// int nValues;
-// int size;
-// ConciseSet[] sets;
-// boolean closedForChange;
-//
-// transient byte[] temp;
-//
-// public BitMapContainer(TableRecordInfoDigest digest, int col) {
-// this.valueLen = digest.length(col);
-// this.size = 0;
-// this.nValues = digest.getMaxID(col) + 1;
-// this.sets = null;
-// this.closedForChange = false;
-//
-// this.temp = new byte[valueLen];
-// }
-//
-// @Override
-// public void append(ImmutableBytesWritable valueBytes) {
-// int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
-// append(value);
-// }
-//
-// public void append(int value) {
-// checkUpdateMode();
-// if (value == Dictionary.NULL_ID[valueLen]) {
-// value = nValues; // set[nValues] holds NULL
-// }
-// sets[value].add(size);
-// size++;
-// }
-//
-// @Override
-// public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
-// int value = getValueIntAt(i);
-// BytesUtil.writeUnsigned(value, temp, 0, valueLen);
-// valueBytes.set(temp, 0, valueLen);
-// }
-//
-// @Override
-// public ConciseSet getBitMap(Integer startId, Integer endId) {
-// if (startId == null && endId == null) {
-// return sets[this.nValues];
-// }
-//
-// int start = 0;
-// int end = this.nValues - 1;
-// if (startId != null) {
-// start = startId;
-// }
-// if (endId != null) {
-// end = endId;
-// }
-//
-// ConciseSet ret = new ConciseSet();
-// for (int i = start; i <= end; ++i) {
-// ConciseSet temp = getBitMap(i);
-// ret.addAll(temp);
-// }
-// return ret;
-// }
-//
-// private ConciseSet getBitMap(int valueId) {
-// if (valueId >= 0 && valueId <= getMaxValueId())
-// return sets[valueId];
-// else
-// return sets[this.nValues];
-// }
-//
-// @Override
-// public int getMaxValueId() {
-// return this.nValues - 1;
-// }
-//
-// public int getValueIntAt(int i) {
-// for (int v = 0; v < nValues; v++) {
-// if (sets[v].contains(i)) {
-// return v;
-// }
-// }
-// // if v is not in [0..nValues-1], then it must be nValue (NULL)
-// return Dictionary.NULL_ID[valueLen];
-// }
-//
-// private void checkUpdateMode() {
-// if (isClosedForChange()) {
-// throw new IllegalStateException();
-// }
-// if (sets == null) {
-// sets = new ConciseSet[nValues + 1];
-// for (int i = 0; i <= nValues; i++) {
-// sets[i] = new ConciseSet();
-// }
-// }
-// }
-//
-// private boolean isClosedForChange() {
-// return closedForChange;
-// }
-//
-// @Override
-// public void closeForChange() {
-// closedForChange = true;
-// }
-//
-// @Override
-// public int getSize() {
-// return size;
-// }
-//
-// public List<ImmutableBytesWritable> toBytes() {
-// if (isClosedForChange() == false)
-// closeForChange();
-//
-// List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
-// for (int i = 0; i <= nValues; i++) {
-// r.add(setToBytes(sets[i]));
-// }
-// return r;
-// }
-//
-// public void fromBytes(List<ImmutableBytesWritable> bytes) {
-// assert nValues + 1 == bytes.size();
-// sets = new ConciseSet[nValues + 1];
-// size = 0;
-// for (int i = 0; i <= nValues; i++) {
-// sets[i] = bytesToSet(bytes.get(i));
-// size += sets[i].size();
-// }
-// closedForChange = true;
-// }
-//
-// private ImmutableBytesWritable setToBytes(ConciseSet set) {
-// byte[] array;
-// if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when
-// // set is empty
-// array = BytesUtil.EMPTY_BYTE_ARRAY;
-// else
-// array = set.toByteBuffer().array();
-// return new ImmutableBytesWritable(array);
-// }
-//
-// private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
-// if (bytes.get() == null || bytes.getLength() == 0) {
-// return new ConciseSet();
-// } else {
-// IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
-// int[] words = new int[intBuffer.capacity()];
-// intBuffer.get(words);
-// return new ConciseSet(words, false);
-// }
-// }
-//
-// @Override
-// public int hashCode() {
-// final int prime = 31;
-// int result = 1;
-// result = prime * result + (closedForChange ? 1231 : 1237);
-// result = prime * result + nValues;
-// result = prime * result + Arrays.hashCode(sets);
-// result = prime * result + size;
-// result = prime * result + valueLen;
-// return result;
-// }
-//
-// @Override
-// public boolean equals(Object obj) {
-// if (this == obj)
-// return true;
-// if (obj == null)
-// return false;
-// if (getClass() != obj.getClass())
-// return false;
-// BitMapContainer other = (BitMapContainer) obj;
-// if (closedForChange != other.closedForChange)
-// return false;
-// if (nValues != other.nValues)
-// return false;
-// if (!Arrays.equals(sets, other.sets))
-// return false;
-// if (size != other.size)
-// return false;
-// if (valueLen != other.valueLen)
-// return false;
-// return true;
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
deleted file mode 100644
index ea35bb4..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * @author yangli9
- */
-public interface ColumnValueContainer {
-
- void append(ImmutableBytesWritable valueBytes);
-
- void closeForChange();
-
- int getSize();
-
- // works only after closeForChange()
- void getValueAt(int i, ImmutableBytesWritable valueBytes);
-
- ConciseSet getBitMap(Integer startId, Integer endId);
-
- int getMaxValueId();
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
deleted file mode 100644
index e395544..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * @author yangli9
- */
-public class CompressedValueContainer implements ColumnValueContainer {
- int valueLen;
- int nValues;
- int cap;
- int size;
- byte[] uncompressed;
- byte[] compressed;
-
- public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) {
- this(digest.length(col), digest.getMaxID(col) + 1, cap);
- }
-
- public CompressedValueContainer(int valueLen, int nValues, int cap) {
- this.valueLen = valueLen;
- this.nValues = nValues;
- this.cap = cap;
- this.size = 0;
- this.uncompressed = null;
- this.compressed = null;
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- checkUpdateMode();
- System.arraycopy(valueBytes.get(), valueBytes.getOffset(), uncompressed, valueLen * size, valueLen);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- valueBytes.set(uncompressed, valueLen * i, valueLen);
- }
-
- @Override
- public ConciseSet getBitMap(Integer startId, Integer endId) {
- ConciseSet ret = new ConciseSet();
- int nullId = Dictionary.NULL_ID[valueLen];
-
- if (startId == null && endId == null) {
- //entry for getting null values
- for (int i = 0; i < size; ++i) {
- int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (nullId == valueID) {
- ret.add(i);
- }
- }
- return ret;
- }
-
- //normal values
- for (int i = 0; i < size; ++i) {
- int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (valueID == nullId) {
- continue;
- }
-
- if (startId != null && valueID < startId) {
- continue;
- }
-
- if (endId != null && valueID > endId) {
- continue;
- }
-
- ret.add(i);
- }
- return ret;
-
- }
-
- @Override
- public int getMaxValueId() {
- return nValues - 1;
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalArgumentException();
- }
- if (uncompressed == null) {
- uncompressed = new byte[valueLen * cap];
- }
- }
-
- private boolean isClosedForChange() {
- return compressed != null;
- }
-
- @Override
- public void closeForChange() {
- checkUpdateMode();
- try {
- compressed = LZFEncoder.encode(uncompressed, 0, valueLen * size);
- } catch (Exception e) {
- throw new RuntimeException("LZF encode failure", e);
- }
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public ImmutableBytesWritable toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
- return new ImmutableBytesWritable(compressed);
- }
-
- public void fromBytes(ImmutableBytesWritable bytes) {
- try {
- uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
- } catch (IOException e) {
- throw new RuntimeException("LZF decode failure", e);
- }
- size = cap = uncompressed.length / valueLen;
- compressed = BytesUtil.EMPTY_BYTE_ARRAY; // mark closed
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + size;
- result = prime * result + valueLen;
- result = prime * result + Arrays.hashCode(uncompressed);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CompressedValueContainer other = (CompressedValueContainer) obj;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- if (!Bytes.equals(uncompressed, 0, size * valueLen, uncompressed, 0, size * valueLen))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java
deleted file mode 100644
index 69e016d..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-/**
- * @author yangli9
- */
-public class IncrementalSliceMaker {
-
- TableRecordInfo info;
- private int nColumns;
- int nRecordsCap;
-
- short shard;
- long sliceTimestamp;
- int nRecords;
- private ColumnValueContainer[] containers;
-
- transient ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- public IncrementalSliceMaker(TableRecordInfo info, short shard) {
- this.info = info;
- this.nColumns = info.getDigest().getColumnCount();
- this.nRecordsCap = Math.max(1, info.getDescriptor().getSliceSize());
-
- this.shard = shard;
- this.sliceTimestamp = Long.MIN_VALUE;
- this.nRecords = 0;
- this.containers = null;
-
- doneSlice(); // init containers
- }
-
- private Slice doneSlice() {
- Slice r = null;
- if (nRecords > 0) {
- for (int i = 0; i < nColumns; i++) {
- containers[i].closeForChange();
- }
- r = new Slice(info.getDigest(), shard, sliceTimestamp, containers);
- }
-
- // reset for next slice
- nRecords = 0;
- containers = new ColumnValueContainer[nColumns];
- // for (int i : info.getDescriptor().getBitmapColumns()) {
- // containers[i] = new CompressedValueContainer(info.getDigest(), i,
- // nRecordsCap);
- // }
- for (int i : info.getDescriptor().getValueColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap);
- }
- for (int i : info.getDescriptor().getMetricsColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap);
- }
-
- return r;
-
- }
-
- // NOTE: record must be appended in time order
- public Slice append(TableRecord rec) {
- if (rec.getShard() != shard)
- throw new IllegalStateException();
-
- Slice doneSlice = null;
-
- if (isFull()) {
- doneSlice = doneSlice();
- }
-
- if (nRecords == 0) {
- sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp());
- }
-
- nRecords++;
- for (int i = 0; i < nColumns; i++) {
- rec.getValueBytes(i, temp);
- containers[i].append(temp);
- }
-
- return doneSlice;
- }
-
- private long increaseSliceTimestamp(long timestamp) {
- if (timestamp < sliceTimestamp)
- throw new IllegalStateException();
-
- if (timestamp == sliceTimestamp)
- return ++timestamp; // ensure slice timestamp increases
- else
- return timestamp;
- }
-
- public Slice close() {
- Slice doneSlice = doneSlice();
- this.sliceTimestamp = Long.MIN_VALUE;
- this.nRecords = 0;
- return doneSlice;
- }
-
- private boolean isFull() {
- return nRecords >= nRecordsCap;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
deleted file mode 100644
index d42cab0..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dimension.DimensionEncoding;
-import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.datatype.LongMutable;
-
-/**
- */
-public class RawTableRecord implements Cloneable {
- TableRecordInfoDigest digest;
- private byte[] buf; // consecutive column value IDs (encoded by dictionary)
-
- public RawTableRecord(TableRecordInfoDigest info) {
- this.digest = info;
- this.buf = new byte[info.getByteFormLen()];
- reset();
- }
-
- public RawTableRecord(RawTableRecord another) {
- this.digest = another.digest;
- this.buf = Bytes.copy(another.buf);
- }
-
- public void reset() {
- Arrays.fill(buf, DimensionEncoding.NULL);
- }
-
- public boolean isMetric(int col) {
- return digest.isMetrics(col);
- }
-
- public FixedLenMeasureCodec<LongMutable> codec(int col) {
- return digest.codec(col);
- }
-
- public final int length(int col) {
- return digest.length(col);
- }
-
- public final int offset(int col) {
- return digest.offset(col);
- }
-
- public int getColumnCount() {
- return digest.getColumnCount();
- }
-
- public void setValueID(int col, int id) {
- BytesUtil.writeUnsigned(id, buf, digest.offset(col), digest.length(col));
- }
-
- public int getValueID(int col) {
- return BytesUtil.readUnsigned(buf, digest.offset(col), digest.length(col));
- }
-
- public void setValueMetrics(int col, LongMutable value) {
- digest.codec(col).write(value, buf, digest.offset(col));
- }
-
- public String getValueMetric(int col) {
- digest.codec(col).read(buf, digest.offset(col));
- return (String) digest.codec(col).getValue();
- }
-
- public byte[] getBytes() {
- return buf;
- }
-
- //TODO is it possible to avoid copying?
- public void setBytes(byte[] bytes, int offset, int length) {
- assert buf.length == length;
- System.arraycopy(bytes, offset, buf, 0, length);
- }
-
- public void setValueBytes(int col, ImmutableBytesWritable bytes) {
- System.arraycopy(bytes.get(), bytes.getOffset(), buf, digest.offset(col), digest.length(col));
- }
-
- public void getValueBytes(int col, ImmutableBytesWritable bytes) {
- bytes.set(buf, offset(col), length(col));
- }
-
- @Override
- public Object clone() {
- return new RawTableRecord(this);
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(buf);
- // result = prime * result + ((digest == null) ? 0 : digest.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- RawTableRecord other = (RawTableRecord) obj;
- if (!Arrays.equals(buf, other.buf))
- return false;
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
deleted file mode 100644
index 45c9c0a..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-public class ShardingSliceBuilder {
-
- IncrementalSliceMaker[] builders;
-
- public ShardingSliceBuilder(TableRecordInfo info) {
- int sharding = info.getDescriptor().getSharding();
- builders = new IncrementalSliceMaker[sharding];
- for (short i = 0; i < sharding; i++) {
- builders[i] = new IncrementalSliceMaker(info, i);
- }
- }
-
- // NOTE: record must be appended in time order
- public Slice append(TableRecord rec) {
- short shard = rec.getShard();
- return builders[shard].append(rec);
- }
-
- public List<Slice> close() {
- List<Slice> result = Lists.newArrayList();
- for (IncrementalSliceMaker builder : builders) {
- Slice slice = builder.close();
- if (slice != null)
- result.add(slice);
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
deleted file mode 100644
index dc2c5c4..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Dictionary;
-
-import com.google.common.base.Objects;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * Within a partition (per timestampGranularity), records are further sliced
- * (per sliceLength) to fit into HBASE cell.
- *
- * @author yangli9
- */
-public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
-
- final TableRecordInfoDigest info;
- final int nColumns;
-
- final short shard;
- final long timestamp;
- final int nRecords;
- final ColumnValueContainer[] containers;
- private Dictionary<?>[] localDictionaries;
-
- public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) {
- this.info = digest;
- this.nColumns = digest.getColumnCount();
-
- this.shard = shard;
- this.timestamp = timestamp;
- this.nRecords = containers[0].getSize();
- this.containers = containers;
-
- assert nColumns == containers.length;
- for (int i = 0; i < nColumns; i++) {
- assert nRecords == containers[i].getSize();
- }
- }
-
- public Dictionary<?>[] getLocalDictionaries() {
- return localDictionaries;
- }
-
- public void setLocalDictionaries(Dictionary<?>[] localDictionaries) {
- this.localDictionaries = localDictionaries;
- }
-
- public int getRecordCount() {
- return this.nRecords;
- }
-
- public short getShard() {
- return shard;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public ColumnValueContainer[] getColumnValueContainers() {
- return containers;
- }
-
- public ColumnValueContainer getColumnValueContainer(int col) {
- return containers[col];
- }
-
- public Iterator<RawTableRecord> iterateWithBitmap(final ConciseSet resultBitMap) {
- if (resultBitMap == null) {
- return this.iterator();
- } else {
- final RawTableRecord rec = info.createTableRecordBytes();
- final ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- return new Iterator<RawTableRecord>() {
- int i = 0;
- int iteratedCount = 0;
- int resultSize = resultBitMap.size();
-
- @Override
- public boolean hasNext() {
- return iteratedCount < resultSize;
- }
-
- @Override
- public RawTableRecord next() {
- while (!resultBitMap.contains(i)) {
- i++;
- }
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- iteratedCount++;
- i++;
-
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
- }
-
- @Override
- public Iterator<RawTableRecord> iterator() {
- return new Iterator<RawTableRecord>() {
- int i = 0;
- RawTableRecord rec = info.createTableRecordBytes();
- ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- @Override
- public boolean hasNext() {
- return i < nRecords;
- }
-
- @Override
- public RawTableRecord next() {
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- i++;
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((info == null) ? 0 : info.hashCode());
- result = prime * result + shard;
- result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- Slice other = (Slice) obj;
- if (shard != other.shard) {
- return false;
- }
- if (timestamp != other.timestamp) {
- return false;
- }
- return Objects.equal(info, other.info);
- }
-
- @Override
- public int compareTo(Slice o) {
- int comp = this.shard - o.shard;
- if (comp != 0)
- return comp;
-
- comp = (int) (this.timestamp - o.timestamp);
- return comp;
- }
-
- public TableRecordInfoDigest getInfo() {
- return info;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
deleted file mode 100644
index 0076919..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.List;
-
-import javax.annotation.Nullable;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.util.IIDictionaryBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-
-/**
- */
-public final class SliceBuilder {
-
- private static Logger logger = LoggerFactory.getLogger(SliceBuilder.class);
-
- private final BatchSliceMaker sliceMaker;
- private final IIDesc iiDesc;
-
- public SliceBuilder(IIDesc desc, short shard) {
- this.iiDesc = desc;
- this.sliceMaker = new BatchSliceMaker(desc, shard);
- }
-
- public Slice buildSlice(StreamingBatch microStreamBatch) throws IOException {
- final List<List<String>> messages = Lists.transform(microStreamBatch.getMessages(), new Function<StreamingMessage, List<String>>() {
- @Nullable
- @Override
- public List<String> apply(@Nullable StreamingMessage input) {
- return input.getData();
- }
- });
- final Dictionary<?>[] dictionaries = IIDictionaryBuilder.buildDictionary(messages, iiDesc);
- TableRecordInfo tableRecordInfo = new TableRecordInfo(iiDesc, dictionaries);
- return build(messages, tableRecordInfo, dictionaries);
- }
-
- private Slice build(List<List<String>> table, final TableRecordInfo tableRecordInfo, Dictionary<?>[] localDictionary) {
- final Slice slice = sliceMaker.makeSlice(tableRecordInfo.getDigest(), Lists.transform(table, new Function<List<String>, TableRecord>() {
- @Nullable
- @Override
- public TableRecord apply(@Nullable List<String> input) {
- TableRecord result = tableRecordInfo.createTableRecord();
- for (int i = 0; i < input.size(); i++) {
- result.setValueString(i, input.get(i));
- }
- return result;
- }
- }));
- slice.setLocalDictionaries(localDictionary);
- return slice;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
deleted file mode 100644
index 3ee34be..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Arrays;
-
-import org.apache.commons.lang.ObjectUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.metadata.datatype.LongMutable;
-
-/**
- * TableRecord extends RawTableRecord by decorating it with a TableRecordInfo.
- *
- * @author yangli9, honma
- */
-public class TableRecord implements Cloneable {
-
- private TableRecordInfo info;
- private RawTableRecord rawRecord;
-
- public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
-
- public TableRecord(RawTableRecord rawRecord, TableRecordInfo info) {
- this.info = info;
- this.rawRecord = rawRecord;
- }
-
- public TableRecord(TableRecord another) {
- this.info = another.info;
- this.rawRecord = (RawTableRecord) another.rawRecord.clone();
- }
-
- public TableRecordInfo getInfo() {
- return info;
- }
-
- @Override
- public Object clone() {
- return new TableRecord(this);
- }
-
- public void reset() {
- rawRecord.reset();
- }
-
- public byte[] getBytes() {
- return rawRecord.getBytes();
- }
-
- public void setBytes(byte[] bytes, int offset, int length) {
- rawRecord.setBytes(bytes, offset, length);
- }
-
- public long getTimestamp() {
- String str = getValueString(info.getTimestampColumn());
- return DateFormat.stringToMillis(str);
- }
-
- public int length(int col) {
- return rawRecord.length(col);
- }
-
- public void setValueStringWithoutDictionary(int col, String value) {
- int offset = info.digest.offset(col);
- int length = info.digest.length(col);
- byte[] src = value.getBytes();
- if (length >= src.length) {
- byte[] dst = rawRecord.getBytes();
- System.arraycopy(src, 0, dst, offset, src.length);
- Arrays.fill(dst, offset + src.length, offset + length, ROWKEY_PLACE_HOLDER_BYTE);
- } else {
- byte[] dst = rawRecord.getBytes();
- System.arraycopy(src, 0, dst, offset, length);
- }
- }
-
- public String getValueStringWithoutDictionary(int col) {
- int offset = info.digest.offset(col);
- int length = info.digest.length(col);
- byte[] bytes = rawRecord.getBytes();
- int i;
- for (i = 0; i < length; ++i) {
- if (bytes[offset + i] == ROWKEY_PLACE_HOLDER_BYTE) {
- break;
- }
- }
- return new String(bytes, offset, i);
- }
-
- public void setValueString(int col, String value) {
- if (rawRecord.isMetric(col)) {
- LongMutable v = rawRecord.codec(col).valueOf(value);
- setValueMetrics(col, v);
- } else {
- final Dictionary<String> dict = info.dict(col);
- if (dict != null) {
- int id = dict.getIdFromValue(value);
- rawRecord.setValueID(col, id);
- } else {
- setValueStringWithoutDictionary(col, value);
- // throw new UnsupportedOperationException("cannot set value when there is no dictionary");
- }
- }
- }
-
- /**
- * get value of columns which belongs to the original table columns.
- * i.e. columns like min_xx, max_yy will never appear
- */
- public String getValueString(int col) {
- if (rawRecord.isMetric(col)) {
- return getValueMetric(col);
- } else {
- final Dictionary<String> dict = info.dict(col);
- if (dict != null) {
- return dict.getValueFromId(rawRecord.getValueID(col));
- } else {
- return getValueStringWithoutDictionary(col);
- // throw new UnsupportedOperationException("cannot get value when there is no dictionary");
- }
- }
- }
-
- public void getValueBytes(int col, ImmutableBytesWritable bytes) {
- rawRecord.getValueBytes(col, bytes);
- }
-
- private void setValueMetrics(int col, LongMutable value) {
- rawRecord.setValueMetrics(col, value);
- }
-
- private String getValueMetric(int col) {
- return rawRecord.getValueMetric(col);
- }
-
- public short getShard() {
- int timestampID = rawRecord.getValueID(info.getTimestampColumn());
- return ShardingHash.getShard(timestampID, info.getDescriptor().getSharding());
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder("[");
- for (int col = 0; col < rawRecord.getColumnCount(); col++) {
- if (col > 0)
- buf.append(",");
- buf.append(getValueString(col));
- }
- buf.append("]");
- return buf.toString();
- }
-
- @Override
- public int hashCode() {
- if (rawRecord != null) {
- return rawRecord.hashCode();
- } else {
- return 0;
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableRecord other = (TableRecord) obj;
- return ObjectUtils.equals(other.rawRecord, this.rawRecord);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
deleted file mode 100644
index 628a08d..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.List;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author yangli9
- * <p/>
- * TableRecordInfo stores application-aware knowledges, while
- * TableRecordInfoDigest only stores byte level knowleges
- */
-public class TableRecordInfo {
-
- final IIDesc desc;
-
- final TableRecordInfoDigest digest;
- final Dictionary<?>[] dictionaries;
-
- public TableRecordInfo(IISegment iiSegment) {
- this(iiSegment.getIIDesc());
- }
-
- public TableRecordInfo(IIDesc desc) {
- this(desc, new Dictionary<?>[desc.listAllColumns().size()]);
- }
-
- public TableRecordInfo(IIDesc desc, Dictionary<?>[] dictionaries) {
- this.desc = desc;
- this.dictionaries = dictionaries;
- this.digest = createDigest(desc, dictionaries);
- }
-
- public TableRecordInfoDigest getDigest() {
- return digest;
- }
-
- private TableRecordInfoDigest createDigest(IIDesc desc, Dictionary<?>[] dictionaryMap) {
- final List<TblColRef> tblColRefs = desc.listAllColumns();
- final int nColumns = tblColRefs.size();
- boolean[] isMetric = new boolean[nColumns];
- int[] lengths = new int[nColumns];
- int[] dictMaxIds = new int[nColumns];
- String[] dataTypes = new String[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- final TblColRef tblColRef = tblColRefs.get(i);
- isMetric[i] = desc.isMetricsCol(i);
- dataTypes[i] = tblColRef.getDatatype();
- if (isMetric[i]) {
- lengths[i] = FixedLenMeasureCodec.get(DataType.getType(tblColRef.getColumnDesc().getDatatype())).getLength();
- } else {
- if (Array.isEmpty(dictionaryMap)) {
- final DataType dataType = DataType.getType(tblColRef.getColumnDesc().getDatatype());
- if (dataType.isNumberFamily()) {
- lengths[i] = 16;
- } else if (dataType.isStringFamily()) {
- lengths[i] = 256;
- } else if (dataType.isDateTimeFamily()) {
- lengths[i] = 19;
- } else {
- throw new RuntimeException("invalid data type:" + dataType);
- }
- dictMaxIds[i] = Integer.MAX_VALUE;
- } else {
- final Dictionary<?> dictionary = dictionaryMap[i];
- lengths[i] = dictionary.getSizeOfId();
- dictMaxIds[i] = dictionary.getMaxId();
- }
- }
- }
- // offsets
- int pos = 0;
- int[] offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += lengths[i];
- }
-
- int byteFormLen = pos;
-
- return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
- }
-
- public TableRecord createTableRecord() {
- return new TableRecord(digest.createTableRecordBytes(), this);
- }
-
- public final IIDesc getDescriptor() {
- return desc;
- }
-
- public final List<TblColRef> getColumns() {
- return desc.listAllColumns();
- }
-
- public int findColumn(TblColRef col) {
- return desc.findColumn(col);
- }
-
- public int findFactTableColumn(String columnName) {
- if (columnName == null)
- return -1;
- for (int i = 0; i < getColumns().size(); ++i) {
- TblColRef tblColRef = getColumns().get(i);
- if (tblColRef.isSameAs(desc.getFactTableName(), columnName)) {
- return i;
- }
- }
- return -1;
- }
-
- // dimensions go with dictionary
- @SuppressWarnings("unchecked")
- public Dictionary<String> dict(int col) {
- // yes, all dictionaries are string based
- return (Dictionary<String>) dictionaries[col];
- }
-
- public int getTimestampColumn() {
- return desc.getTimestampColumn();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
deleted file mode 100644
index 9eebdbe..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.index;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
-
-import com.google.common.base.Objects;
-
-/**
- */
-public class TableRecordInfoDigest {
-
- private String[] metricDataTypes;
- private int nColumns;
- private int byteFormLen;
-
- private int[] offsets;// column offset in byte form row
- private int[] dictMaxIds;// max id for each of the dict
- private int[] lengths;// length of each encoded dict
- private boolean[] isMetric;// whether it's metric or dict
- private FixedLenMeasureCodec<?>[] measureCodecs;
-
- public TableRecordInfoDigest(int nColumns, int byteFormLen, //
- int[] offsets, int[] dictMaxIds, int[] lengths, boolean[] isMetric, String[] metricDataTypes) {
- this.nColumns = nColumns;
- this.byteFormLen = byteFormLen;
- this.offsets = offsets;
- this.dictMaxIds = dictMaxIds;
- this.lengths = lengths;
- this.isMetric = isMetric;
- this.metricDataTypes = metricDataTypes;
- this.measureCodecs = new FixedLenMeasureCodec[nColumns];
- for (int i = 0; i < isMetric.length; i++) {
- if (isMetric[i]) {
- measureCodecs[i] = FixedLenMeasureCodec.get(DataType.getType(metricDataTypes[i]));
- }
- }
- }
-
- private TableRecordInfoDigest() {
- }
-
- public int getByteFormLen() {
- return byteFormLen;
- }
-
- public boolean isMetrics(int col) {
- return isMetric[col];
- }
-
- public boolean[] isMetrics() {
- return isMetric;
- }
-
- public int getColumnCount() {
- return nColumns;
- }
-
- public int offset(int col) {
- return offsets[col];
- }
-
- public int length(int col) {
- return lengths[col];
- }
-
- public int getMaxID(int col) {
- return dictMaxIds[col];
- }
-
- public boolean[] getIsMetric() {
- return isMetric;
- }
-
- public String[] getMetricDataTypes() {
- return metricDataTypes;
- }
-
- public RawTableRecord createTableRecordBytes() {
- return new RawTableRecord(this);
- }
-
- @Override
- public int hashCode() {
- return com.google.common.base.Objects.hashCode(nColumns, offsets, dictMaxIds, lengths, isMetric, metricDataTypes);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj instanceof TableRecordInfoDigest) {
- TableRecordInfoDigest other = (TableRecordInfoDigest) obj;
- return Objects.equal(this.nColumns, other.nColumns) && Arrays.equals(this.offsets, other.offsets) && Arrays.equals(this.dictMaxIds, other.dictMaxIds) && Arrays.equals(this.lengths, other.lengths) && Arrays.equals(this.isMetric, other.isMetric) && Arrays.equals(this.metricDataTypes, other.metricDataTypes);
- } else {
- return false;
- }
- }
-
- // metrics go with fixed-len codec
- @SuppressWarnings("unchecked")
- public FixedLenMeasureCodec<LongMutable> codec(int col) {
- // yes, all metrics are long currently
- return (FixedLenMeasureCodec<LongMutable>) measureCodecs[col];
- }
-
- public static byte[] serialize(TableRecordInfoDigest o) {
- ByteBuffer buf = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static TableRecordInfoDigest deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- public static TableRecordInfoDigest deserialize(ByteBuffer buffer) {
- return serializer.deserialize(buffer);
- }
-
- private static final BytesSerializer<TableRecordInfoDigest> serializer = new BytesSerializer<TableRecordInfoDigest>() {
-
- @Override
- public void serialize(TableRecordInfoDigest value, ByteBuffer out) {
- BytesUtil.writeVInt(value.nColumns, out);
- BytesUtil.writeVInt(value.byteFormLen, out);
- BytesUtil.writeIntArray(value.offsets, out);
- BytesUtil.writeIntArray(value.dictMaxIds, out);
- BytesUtil.writeIntArray(value.lengths, out);
- BytesUtil.writeBooleanArray(value.isMetric, out);
- BytesUtil.writeAsciiStringArray(value.metricDataTypes, out);
-
- }
-
- @Override
- public TableRecordInfoDigest deserialize(ByteBuffer in) {
- TableRecordInfoDigest result = new TableRecordInfoDigest();
- result.nColumns = BytesUtil.readVInt(in);
- result.byteFormLen = BytesUtil.readVInt(in);
- result.offsets = BytesUtil.readIntArray(in);
- result.dictMaxIds = BytesUtil.readIntArray(in);
- result.lengths = BytesUtil.readIntArray(in);
- result.isMetric = BytesUtil.readBooleanArray(in);
- result.metricDataTypes = BytesUtil.readAsciiStringArray(in);
- return result;
- }
-
- };
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
deleted file mode 100644
index 0ec2638..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.invertedindex.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.datatype.DataType;
-
-/**
- */
-public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> {
-
- private DataType type;
- private int presision;
- private HyperLogLogPlusCounter current;
-
- public FixedHLLCodec(DataType type) {
- this.type = type;
- this.presision = type.getPrecision();
- this.current = new HyperLogLogPlusCounter(this.presision);
- }
-
- @Override
- public int getLength() {
- return 1 << presision;
- }
-
- @Override
- public DataType getDataType() {
- return type;
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(String value) {
- current.clear();
- if (value != null)
- current.add(value.getBytes());
- return current;
- }
-
- @Override
- public Object getValue() {
- return current;
- }
-
- @Override
- public HyperLogLogPlusCounter read(byte[] buf, int offset) {
- current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
- return current;
- }
-
- @Override
- public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
- v.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
- }
-
- @Override
- public HyperLogLogPlusCounter read(ByteBuffer buffer) {
- current.readRegistersArray(buffer);
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
deleted file mode 100644
index bd952a1..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.measure.hllc.HLLCMeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
-abstract public class FixedLenMeasureCodec<T> {
-
- public static FixedLenMeasureCodec<?> get(DataType type) {
- if (HLLCMeasureType.DATATYPE_HLLC.equals(type.getName())) {
- return new FixedHLLCodec(type);
- } else {
- return new FixedPointLongCodec(type);
- }
- }
-
- abstract public int getLength();
-
- abstract public DataType getDataType();
-
- abstract public T valueOf(String value);
-
- abstract public Object getValue();
-
- abstract public T read(byte[] buf, int offset);
-
- abstract public void write(T v, byte[] buf, int offset);
-
- abstract public T read(ByteBuffer buffer);
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
deleted file mode 100644
index 67e5158..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
-
-public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
-
- private static final int SIZE = 8;
- // number of digits after decimal point
- int scale;
- DataType type;
- // avoid massive object creation
- LongMutable current = new LongMutable();
-
- public FixedPointLongCodec(DataType type) {
- this.type = type;
- this.scale = Math.max(0, type.getScale());
- }
-
- @Override
- public int getLength() {
- return SIZE;
- }
-
- @Override
- public DataType getDataType() {
- return type;
- }
-
- long getValueIgnoringDecimalPoint(String value) {
- int index = value.indexOf('.');
-
- if (index == 0 || index == value.length() - 1) {
- throw new RuntimeException("Bad decimal format: " + value);
- } else if (index < 0) {
- return Long.valueOf(value) * (int) Math.pow(10, scale);
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append(value.substring(0, index));
-
- //if there are more than scale digits after the decimal point, the tail will be discarded
- int end = Math.min(value.length(), index + scale + 1);
- sb.append(value.substring(index + 1, end));
- int diff = index + scale + 1 - value.length();
- //if there are less than scale digits after the decimal point, the tail will be compensated
- for (int i = 0; i < diff; i++) {
- sb.append('0');
- }
- return Long.valueOf(sb.toString());
- }
- }
-
- String restoreDecimalPoint(long value) {
- if (scale < 0) {
- throw new RuntimeException("Bad scale: " + scale + " with value: " + value);
- } else if (scale == 0) {
- return Long.toString(value);
- } else {
- return String.format("%." + scale + "f", value / (Math.pow(10, scale)));
- }
- }
-
- @Override
- public LongMutable valueOf(String value) {
- if (value == null)
- current.set(0L);
- else
- current.set(getValueIgnoringDecimalPoint(value));
- return current;
- }
-
- @Override
- public String getValue() {
- if (scale == 0)
- return current.toString();
- else
- return restoreDecimalPoint(current.get());
- }
-
- @Override
- public LongMutable read(byte[] buf, int offset) {
- current.set(BytesUtil.readLong(buf, offset, SIZE));
- return current;
- }
-
- @Override
- public void write(LongMutable v, byte[] buf, int offset) {
- BytesUtil.writeLong(v == null ? 0 : v.get(), buf, offset, SIZE);
- }
-
- @Override
- public LongMutable read(ByteBuffer buffer) {
- current.set(BytesUtil.readLong(buffer, SIZE));
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
deleted file mode 100644
index 66fb67d..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java
+++ /dev/null
@@ -1,418 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.model;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.commons.net.util.Base64;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.IEngineAware;
-import org.apache.kylin.metadata.model.IStorageAware;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ModelDimensionDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings("serial")
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class IIDesc extends RootPersistentEntity {
-
- public static final String HBASE_FAMILY = "f";
- public static final String HBASE_QUALIFIER = "c";
- public static final String HBASE_DICTIONARY = "d";
- public static final byte[] HBASE_FAMILY_BYTES = Bytes.toBytes(HBASE_FAMILY);
- public static final byte[] HBASE_QUALIFIER_BYTES = Bytes.toBytes(HBASE_QUALIFIER);
- public static final byte[] HBASE_DICTIONARY_BYTES = Bytes.toBytes(HBASE_DICTIONARY);
-
- private KylinConfig config;
- private DataModelDesc model;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("model_name")
- private String modelName;
- @JsonProperty("timestamp_dimension")
- private String timestampDimension;
- @JsonProperty("bitmap_dimensions")
- private List<ModelDimensionDesc> bitmapDimensions = Collections.emptyList();
- @JsonProperty("value_dimensions")
- private List<ModelDimensionDesc> valueDimensions;
- @JsonProperty("metrics")
- private String[] metricNames;
- @JsonProperty("sharding")
- private short sharding = 1; // parallelism
- @JsonProperty("slice_size")
- private int sliceSize = 50000; // no. rows
-
- @JsonProperty("engine_type")
- private int engineType = IEngineAware.ID_MR_II;
-
- @JsonProperty("storage_type")
- private int storageType = IStorageAware.ID_HBASE;
-
- @JsonProperty("signature")
- private String signature;
-
- // computed
- private List<TableDesc> allTables = Lists.newArrayList();
- private List<TblColRef> allColumns = Lists.newArrayList();
- private List<TblColRef> allDimensions = Lists.newArrayList();
- private int tsCol;
- private int[] bitmapCols;
- private int[] valueCols;
- private int[] metricsCols;
- private BitSet metricsColSet;
- private List<MeasureDesc> measureDescs;
-
- public void init(MetadataManager metadataManager) {
-
- config = metadataManager.getConfig();
-
- if (this.modelName == null || this.modelName.length() == 0) {
- throw new RuntimeException("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
- }
-
- this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
-
- if (this.model == null) {
- throw new RuntimeException("No data model found with name '" + modelName + "'.");
- }
-
- timestampDimension = timestampDimension.toUpperCase();
-
- // capitalize
- ModelDimensionDesc.capicalizeStrings(bitmapDimensions);
- ModelDimensionDesc.capicalizeStrings(valueDimensions);
- StringUtil.toUpperCaseArray(metricNames, metricNames);
-
- // retrieve all columns and all tables, and make available measure to ii
- HashSet<String> allTableNames = Sets.newHashSet();
- measureDescs = Lists.newArrayList();
- measureDescs.add(makeCountMeasure());
- for (ModelDimensionDesc modelDimensionDesc : Iterables.concat(bitmapDimensions, valueDimensions)) {
- TableDesc tableDesc = this.getTableDesc(modelDimensionDesc.getTable());
- for (String column : modelDimensionDesc.getColumns()) {
- ColumnDesc columnDesc = tableDesc.findColumnByName(column);
- TblColRef tcr = columnDesc.getRef();
- allColumns.add(tcr);
- allDimensions.add(tcr);
- measureDescs.add(makeHLLMeasure(columnDesc, "hllc10"));
- }
-
- if (!allTableNames.contains(tableDesc.getIdentity())) {
- allTableNames.add(tableDesc.getIdentity());
- allTables.add(tableDesc);
- }
- }
- for (String column : metricNames) {
- TableDesc tableDesc = this.getTableDesc(this.getFactTableName());
- ColumnDesc columnDesc = tableDesc.findColumnByName(column);
- allColumns.add(columnDesc.getRef());
- measureDescs.add(makeNormalMeasure("SUM", columnDesc));
- measureDescs.add(makeNormalMeasure("MIN", columnDesc));
- measureDescs.add(makeNormalMeasure("MAX", columnDesc));
- if (!allTableNames.contains(tableDesc.getIdentity())) {
- allTableNames.add(tableDesc.getIdentity());
- allTables.add(tableDesc);
- }
- }
-
- // indexing for each type of columns
- bitmapCols = new int[ModelDimensionDesc.getColumnCount(bitmapDimensions)];
- valueCols = new int[ModelDimensionDesc.getColumnCount(valueDimensions)];
- metricsCols = new int[metricNames.length];
- metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount());
-
- int totalIndex = 0;
- for (int i = 0; i < bitmapCols.length; ++i, ++totalIndex) {
- bitmapCols[i] = totalIndex;
- }
- for (int i = 0; i < valueCols.length; ++i, ++totalIndex) {
- valueCols[i] = totalIndex;
- }
- for (int i = 0; i < metricsCols.length; ++i, ++totalIndex) {
- metricsCols[i] = totalIndex;
- metricsColSet.set(totalIndex);
- }
-
- // partitioning column
- tsCol = -1;
- for (int i = 0; i < allColumns.size(); ++i) {
- TblColRef col = allColumns.get(i);
-
- if (col.isSameAs(this.getFactTableName(), this.timestampDimension)) {
- tsCol = i;
- break;
- }
- }
- if (tsCol < 0)
- throw new RuntimeException("timestamp_dimension is not in bitmapDimensions or valueDimensions");
- }
-
- private TableDesc getTableDesc(String tableName) {
- return MetadataManager.getInstance(this.config).getTableDesc(tableName);
- }
-
- public String getResourcePath() {
- return getIIDescResourcePath(name);
- }
-
- public static String getIIDescResourcePath(String descName) {
- return ResourceStore.II_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
- }
-
- public List<MeasureDesc> getMeasures() {
- return measureDescs;
- }
-
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measureDescs) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
- private MeasureDesc makeNormalMeasure(String func, ColumnDesc columnDesc) {
- String columnName = columnDesc.getName();
- String returnType = columnDesc.getTypeName();
- MeasureDesc measureDesc = new MeasureDesc();
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression(func);
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue(columnName);
- p1.setColRefs(ImmutableList.of(columnDesc.getRef()));
- f1.setParameter(p1);
- f1.setReturnType(returnType);
- if (f1.isSum() && f1.getReturnDataType().isIntegerFamily()) {
- f1.setReturnType("bigint");
- }
-
- measureDesc.setFunction(f1);
- measureDesc.setName(func + "_" + columnName);
- return measureDesc;
- }
-
- /**
- *
- * @param hllType represents the precision
- */
- private MeasureDesc makeHLLMeasure(ColumnDesc columnDesc, String hllType) {
- String columnName = columnDesc.getName();
- MeasureDesc measureDesc = new MeasureDesc();
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("COUNT_DISTINCT");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue(columnName);
- p1.setColRefs(ImmutableList.of(columnDesc.getRef()));
- f1.setParameter(p1);
- f1.setReturnType(hllType);
- measureDesc.setFunction(f1);
- measureDesc.setName("COUNT_DISTINCT" + "_" + columnName);
- return measureDesc;
- }
-
- private MeasureDesc makeCountMeasure() {
- MeasureDesc measureDesc = new MeasureDesc();
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("COUNT");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("constant");
- p1.setValue("1");
- f1.setParameter(p1);
- f1.setReturnType("bigint");
- measureDesc.setFunction(f1);
- measureDesc.setName("COUNT_1");
- return measureDesc;
- }
-
- /**
- * at first stage the only table in II is fact table, tables
- *
- * @return
- */
- public List<TableDesc> listTables() {
- return allTables;
- }
-
- public List<TblColRef> listAllColumns() {
- return allColumns;
- }
-
- public List<TblColRef> listAllDimensions() {
- return allDimensions;
- }
-
- public TblColRef findColumnRef(String table, String column) {
- ColumnDesc columnDesc = this.getTableDesc(table).findColumnByName(column);
- return columnDesc.getRef();
- }
-
- public int findColumn(TblColRef col) {
- return this.allColumns.indexOf(col);
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public String getName() {
- return name;
- }
-
- public String getModelName() {
- return modelName;
- }
-
- public void setModelName(String modelName) {
- this.modelName = modelName;
- }
-
- public DataModelDesc getModel() {
- return model;
- }
-
- public void setModel(DataModelDesc model) {
- this.model = model;
- }
-
- public int getTimestampColumn() {
- return tsCol;
- }
-
- public int[] getBitmapColumns() {
- return bitmapCols;
- }
-
- public int[] getValueColumns() {
- return valueCols;
- }
-
- public int[] getMetricsColumns() {
- return metricsCols;
- }
-
- public short getSharding() {
- return sharding;
- }
-
- public int getSliceSize() {
- return sliceSize;
- }
-
- public String getSignature() {
- return signature;
- }
-
- public void setSignature(String signature) {
- this.signature = signature;
- }
-
- public boolean isMetricsCol(TblColRef col) {
- if (!col.getTable().equalsIgnoreCase(this.getFactTableName()))
- return false;
- return isMetricsCol(this.findColumn(col));
- }
-
- public boolean isMetricsCol(int index) {
- return metricsColSet.get(index);
- }
-
- /**
- * the returned fact table name is guaranteed to be in the form of db.table
- *
- * @return
- */
- public String getFactTableName() {
- return this.model.getFactTable().toUpperCase();
- }
-
- public String getTimestampDimension() {
- return timestampDimension;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String calculateSignature() {
- MessageDigest md = null;
- try {
- md = MessageDigest.getInstance("MD5");
- StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTableName()).append("|").append(timestampDimension).append("|").append(JsonUtil.writeValueAsString(this.bitmapDimensions)).append("|").append(JsonUtil.writeValueAsString(valueDimensions)).append("|").append(JsonUtil.writeValueAsString(this.metricNames)).append("|").append(sharding).append("|").append(sliceSize);
-
- byte[] signature = md.digest(sigString.toString().getBytes());
- return new String(Base64.encodeBase64(signature));
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException("Failed to calculate signature");
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Failed to calculate signature");
- }
-
- }
-
- public int getStorageType() {
- return storageType;
- }
-
- public void setStorageType(int storageType) {
- this.storageType = storageType;
- }
-
- public int getEngineType() {
- return engineType;
- }
-
- public void setEngineType(int engineType) {
- this.engineType = engineType;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
deleted file mode 100644
index 21e5677..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.List;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.IntermediateColumnDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc {
-
- private IIDesc iiDesc;
- private String tableName;
- private List<IntermediateColumnDesc> columnList = Lists.newArrayList();
-
- public IIJoinedFlatTableDesc(IIDesc iiDesc) {
- this.iiDesc = iiDesc;
- parseIIDesc();
- }
-
- private void parseIIDesc() {
- this.tableName = "kylin_intermediate_ii_" + iiDesc.getName();
-
- int columnIndex = 0;
- for (TblColRef col : iiDesc.listAllColumns()) {
- columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col));
- columnIndex++;
- }
- }
-
- @Override
- public String getTableName() {
- return tableName + "_" + "II_Flat";
- }
-
- @Override
- public List<IntermediateColumnDesc> getColumnList() {
- return columnList;
- }
-
- @Override
- public DataModelDesc getDataModel() {
- return iiDesc.getModel();
- }
-
-}