You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/10 07:48:33 UTC
[40/54] [abbrv] [partial] incubator-kylin git commit: cleanup for
migration from github.com
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfo.java b/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfo.java
deleted file mode 100644
index b831328..0000000
--- a/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfo.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.measure.fixedlen.FixedLenMeasureCodec;
-import com.kylinolap.dict.Dictionary;
-import com.kylinolap.dict.DictionaryManager;
-import com.kylinolap.metadata.model.cube.TblColRef;
-import com.kylinolap.metadata.model.invertedindex.InvertedIndexDesc;
-import com.kylinolap.metadata.model.schema.ColumnDesc;
-import com.kylinolap.metadata.model.schema.TableDesc;
-
-/**
- * @author yangli9
- * <p/>
- * TableRecordInfo stores application-aware knowledges,
- * while TableRecordInfoDigest only stores byte level knowleges
- */
-public class TableRecordInfo extends TableRecordInfoDigest {
-
- final CubeSegment seg;
- final InvertedIndexDesc desc;
- final TableDesc tableDesc;
-
- final String[] colNames;
- final Dictionary<?>[] dictionaries;
- final FixedLenMeasureCodec<?>[] measureSerializers;
-
-
- public TableRecordInfo(CubeSegment cubeSeg) throws IOException {
-
- seg = cubeSeg;
- desc = seg.getCubeInstance().getInvertedIndexDesc();
- tableDesc = desc.getFactTableDesc();
-
- nColumns = tableDesc.getColumnCount();
- colNames = new String[nColumns];
- dictionaries = new Dictionary<?>[nColumns];
- measureSerializers = new FixedLenMeasureCodec<?>[nColumns];
-
- DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig());
- for (ColumnDesc col : tableDesc.getColumns()) {
- int i = col.getZeroBasedIndex();
- colNames[i] = col.getName();
- if (desc.isMetricsCol(i)) {
- measureSerializers[i] = FixedLenMeasureCodec.get(col.getType());
- } else {
- String dictPath = seg.getDictResPath(new TblColRef(col));
- dictionaries[i] = dictMgr.getDictionary(dictPath);
- }
- }
-
- //isMetric
- isMetric = new boolean[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- isMetric[i] = desc.isMetricsCol(i);
- }
-
- //lengths
- lengths = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- lengths[i] = isMetrics(i) ? measureSerializers[i].getLength() : dictionaries[i].getSizeOfId();
- }
-
- //dict max id
- dictMaxIds = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- if (!isMetrics(i))
- dictMaxIds[i] = dictionaries[i].getMaxId();
- }
-
- //offsets
- int pos = 0;
- offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += length(i);
- }
-
- byteFormLen = pos;
- }
-
- @Override
- public TableRecordBytes createTableRecord() {
- return new TableRecord(this);
- }
-
- public InvertedIndexDesc getDescriptor() {
- return desc;
- }
-
- public ColumnDesc[] getColumns() {
- return tableDesc.getColumns();
- }
-
-
- // dimensions go with dictionary
- @SuppressWarnings("unchecked")
- public Dictionary<String> dict(int col) {
- // yes, all dictionaries are string based
- return (Dictionary<String>) dictionaries[col];
- }
-
- // metrics go with fixed-len codec
- @SuppressWarnings("unchecked")
- public FixedLenMeasureCodec<LongWritable> codec(int col) {
- // yes, all metrics are long currently
- return (FixedLenMeasureCodec<LongWritable>) measureSerializers[col];
- }
-
-
- public int getTimestampColumn() {
- return desc.getTimestampColumn();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((seg == null) ? 0 : seg.hashCode());
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableRecordInfo other = (TableRecordInfo) obj;
- if (seg == null) {
- if (other.seg != null)
- return false;
- } else if (!seg.equals(other.seg))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfoDigest.java b/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfoDigest.java
deleted file mode 100644
index 2833183..0000000
--- a/cube/src/main/java/com/kylinolap/cube/invertedindex/TableRecordInfoDigest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-package com.kylinolap.cube.invertedindex;
-
-
-import com.kylinolap.common.util.BytesSerializer;
-import com.kylinolap.common.util.BytesUtil;
-
-import java.nio.ByteBuffer;
-
-/**
- * Created by honma on 11/10/14.
- */
-public class TableRecordInfoDigest implements TableRecordFactory {
-
- protected int nColumns;
- protected int byteFormLen;
-
- protected int[] offsets;//column offset in byte form row
- protected int[] dictMaxIds;//max id for each of the dict
- protected int[] lengths;//length of each encoded dict
- protected boolean[] isMetric;//whether it's metric or dict
-
- public boolean isMetrics(int col) {
- return isMetric[col];
- }
-
- public int getColumnCount() {
- return nColumns;
- }
-
- public int offset(int col) {
- return offsets[col];
- }
-
- public int length(int col) {
- return lengths[col];
- }
-
- public int getMaxID(int col) {
- return dictMaxIds[col];
- }
-
- @Override
- public TableRecordBytes createTableRecord() {
- return new TableRecordBytes(this);
- }
-
- public static byte[] serialize(TableRecordInfoDigest o) {
- ByteBuffer buf = ByteBuffer.allocate(Serializer.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static TableRecordInfoDigest deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- public static TableRecordInfoDigest deserialize(ByteBuffer buffer) {
- return serializer.deserialize(buffer);
- }
-
- private static final Serializer serializer = new Serializer();
-
- private static class Serializer implements BytesSerializer<TableRecordInfoDigest> {
-
- @Override
- public void serialize(TableRecordInfoDigest value, ByteBuffer out) {
- BytesUtil.writeVInt(value.nColumns, out);
- BytesUtil.writeVInt(value.byteFormLen, out);
- BytesUtil.writeIntArray(value.offsets, out);
- BytesUtil.writeIntArray(value.dictMaxIds, out);
- BytesUtil.writeIntArray(value.lengths, out);
- }
-
- @Override
- public TableRecordInfoDigest deserialize(ByteBuffer in) {
- TableRecordInfoDigest result = new TableRecordInfoDigest();
- result.nColumns = BytesUtil.readVInt(in);
- result.byteFormLen = BytesUtil.readVInt(in);
- result.offsets = BytesUtil.readIntArray(in);
- result.dictMaxIds = BytesUtil.readIntArray(in);
- result.lengths = BytesUtil.readIntArray(in);
- return result;
- }
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/AbstractRowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/AbstractRowKeyEncoder.java b/cube/src/main/java/com/kylinolap/cube/kv/AbstractRowKeyEncoder.java
deleted file mode 100644
index e2a367c..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/AbstractRowKeyEncoder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.cuboid.Cuboid;
-import com.kylinolap.dict.Dictionary;
-import com.kylinolap.metadata.model.cube.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public abstract class AbstractRowKeyEncoder {
-
- public static final byte DEFAULT_BLANK_BYTE = Dictionary.NULL;
-
- protected static final Logger logger = LoggerFactory.getLogger(AbstractRowKeyEncoder.class);
-
- private static final Map<String, Map<Long, AbstractRowKeyEncoder>> ENCODER_CACHE = new ConcurrentHashMap<String, Map<Long, AbstractRowKeyEncoder>>();
-
- public static AbstractRowKeyEncoder createInstance(CubeSegment cubeSeg, Cuboid cuboid) {
-
- // The storage location identifier is unique for every segment
- Map<Long, AbstractRowKeyEncoder> cubeCache = ENCODER_CACHE.get(cubeSeg.getStorageLocationIdentifier());
-
- if (cubeCache == null) {
- cubeCache = new HashMap<Long, AbstractRowKeyEncoder>();
- ENCODER_CACHE.put(cuboid.getCube().getName(), cubeCache);
- }
-
- AbstractRowKeyEncoder encoder = cubeCache.get(cuboid.getId());
- if (encoder == null) {
- encoder = new RowKeyEncoder(cubeSeg, cuboid);
- cubeCache.put(cuboid.getId(), encoder);
- }
- return encoder;
- }
-
- protected final Cuboid cuboid;
- protected byte blankByte = DEFAULT_BLANK_BYTE;
-
- protected AbstractRowKeyEncoder(Cuboid cuboid) {
- this.cuboid = cuboid;
- }
-
- public void setBlankByte(byte blankByte) {
- this.blankByte = blankByte;
- }
-
- abstract public byte[] encode(Map<TblColRef, String> valueMap);
-
- abstract public byte[] encode(byte[][] values);
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/FuzzyKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/FuzzyKeyEncoder.java b/cube/src/main/java/com/kylinolap/cube/kv/FuzzyKeyEncoder.java
deleted file mode 100644
index 172374e..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/FuzzyKeyEncoder.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import java.util.Arrays;
-
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.cuboid.Cuboid;
-
-/**
- *
- * @author xjiang
- *
- */
-public class FuzzyKeyEncoder extends RowKeyEncoder {
-
- public FuzzyKeyEncoder(CubeSegment seg, Cuboid cuboid) {
- super(seg, cuboid);
- }
-
- @Override
- protected byte[] defaultValue(int length) {
- byte[] keyBytes = new byte[length];
- Arrays.fill(keyBytes, RowConstants.FUZZY_MASK_ZERO);
- return keyBytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/FuzzyMaskEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/FuzzyMaskEncoder.java b/cube/src/main/java/com/kylinolap/cube/kv/FuzzyMaskEncoder.java
deleted file mode 100644
index e42ba1e..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/FuzzyMaskEncoder.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import java.util.Arrays;
-
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.cuboid.Cuboid;
-import com.kylinolap.metadata.model.cube.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public class FuzzyMaskEncoder extends RowKeyEncoder {
-
- public FuzzyMaskEncoder(CubeSegment seg, Cuboid cuboid) {
- super(seg, cuboid);
- }
-
- @Override
- protected int fillHeader(byte[] bytes, byte[][] values) {
- // always fuzzy match cuboid ID to lock on the selected cuboid
- int cuboidStart = this.headerLength - RowConstants.ROWKEY_CUBOIDID_LEN;
- Arrays.fill(bytes, 0, cuboidStart, RowConstants.FUZZY_MASK_ONE);
- Arrays.fill(bytes, cuboidStart, this.headerLength, RowConstants.FUZZY_MASK_ZERO);
- return this.headerLength;
- }
-
- @Override
- protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) {
- if (value == null) {
- Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, RowConstants.FUZZY_MASK_ONE);
- } else {
- Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, RowConstants.FUZZY_MASK_ZERO);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/RowConstants.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/RowConstants.java b/cube/src/main/java/com/kylinolap/cube/kv/RowConstants.java
deleted file mode 100644
index fc196fe..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/RowConstants.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-/**
- *
- * @author xjiang
- *
- */
-public class RowConstants {
-
- // row key fixed length place holder
- public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
- // row key lower bound
- public static final byte ROWKEY_LOWER_BYTE = 0;
- // row key upper bound
- public static final byte ROWKEY_UPPER_BYTE = (byte) 0xff;
- // row key cuboid id length
- public static final int ROWKEY_CUBOIDID_LEN = 8;
-
- // fuzzy mask
- public static final byte FUZZY_MASK_ZERO = 0;
- public static final byte FUZZY_MASK_ONE = 1;
-
- // row value delimiter
- public static final byte ROWVALUE_DELIMITER_BYTE = 7;
- public static final String ROWVALUE_DELIMITER_STRING = String.valueOf((char) 7);
- public static final byte[] ROWVALUE_DELIMITER_BYTES = { 7 };
-
- public static final int ROWVALUE_BUFFER_SIZE = 1024 * 1024; // 1 MB
-
- // marker class
- public static final byte[][] BYTE_ARR_MARKER = new byte[0][];
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnIO.java b/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnIO.java
deleted file mode 100644
index d064769..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnIO.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.kv;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.common.util.BytesUtil;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.dict.Dictionary;
-import com.kylinolap.metadata.model.cube.RowKeyDesc;
-import com.kylinolap.metadata.model.cube.TblColRef;
-
-/**
- * Read/Write column values from/into bytes
- *
- * @author yangli9
- */
-@SuppressWarnings("unchecked")
-public class RowKeyColumnIO {
-
- private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class);
-
- private CubeSegment seg;
- private RowKeyDesc rowkeyDesc;
- private boolean forceNoDict = Boolean.getBoolean("forceNoDict");
-
- public RowKeyColumnIO(CubeSegment cubeSeg) {
- this.seg = cubeSeg;
- this.rowkeyDesc = seg.getCubeDesc().getRowkey();
- }
-
- public CubeSegment getCubeSegment() {
- return seg;
- }
-
- public int getColumnLength(TblColRef col) {
- Dictionary<String> dict = getDictionary(col);
- if (dict == null) {
- return rowkeyDesc.getColumnLength(col);
- } else {
- return dict.getSizeOfId();
- }
- }
-
- public void writeColumn(TblColRef column, byte[] value, int valueLen, byte dft, byte[] output, int outputOffset) {
- writeColumn(column, value, valueLen, 0, dft, output, outputOffset);
- }
-
- public void writeColumn(TblColRef column, byte[] value, int valueLen, int roundingFlag, byte dft, byte[] output, int outputOffset) {
-
- Dictionary<String> dict = getDictionary(column);
- int columnLen = getColumnLength(column);
-
- // non-dict value
- if (dict == null) {
- byte[] valueBytes = padFixLen(columnLen, value);
- System.arraycopy(valueBytes, 0, output, outputOffset, columnLen);
- return;
- }
-
- // dict value
- try {
- int id = dict.getIdFromValueBytes(value, 0, valueLen, roundingFlag);
- BytesUtil.writeUnsigned(id, output, outputOffset, dict.getSizeOfId());
- } catch (IllegalArgumentException ex) {
- for (int i = outputOffset; i < outputOffset + columnLen; i++)
- output[i] = dft;
- logger.error("Can't translate value " + Bytes.toString(value, 0, valueLen) + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", dft));
- }
- }
-
- private byte[] padFixLen(int length, byte[] valueBytes) {
- int valLen = valueBytes.length;
- if (valLen == length) {
- return valueBytes;
- } else if (valLen < length) {
- byte[] newValueBytes = new byte[length];
- System.arraycopy(valueBytes, 0, newValueBytes, 0, valLen);
- Arrays.fill(newValueBytes, valLen, length, RowConstants.ROWKEY_PLACE_HOLDER_BYTE);
- return newValueBytes;
- } else {
- return Arrays.copyOf(valueBytes, length);
- }
- }
-
- public String readColumnString(TblColRef col, byte[] bytes, int bytesLen) {
- Dictionary<String> dict = getDictionary(col);
- if (dict == null) {
- bytes = Bytes.head(bytes, bytesLen);
- if (isNull(bytes)) {
- return null;
- }
- bytes = removeFixLenPad(bytes, 0);
- return Bytes.toString(bytes);
- } else {
- int id = BytesUtil.readUnsigned(bytes, 0, bytesLen);
- try {
- String value = dict.getValueFromId(id);
- return value;
- } catch (IllegalArgumentException e) {
- logger.error("Can't get dictionary value for column " + col.getName() + " (id = " + id + ")");
- return "";
- }
- }
- }
-
- private boolean isNull(byte[] bytes) {
- // all 0xFF is NULL
- if (bytes.length == 0)
- return false;
- for (int i = 0; i < bytes.length; i++) {
- if (bytes[i] != AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE)
- return false;
- }
- return true;
- }
-
- private byte[] removeFixLenPad(byte[] bytes, int offset) {
- int padCount = 0;
- for (int i = offset; i < bytes.length; i++) {
- byte vb = bytes[i];
- if (vb == RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
- padCount++;
- }
- }
-
- int size = bytes.length - offset - padCount;
- byte[] stripBytes = new byte[size];
- int index = 0;
- for (int i = offset; i < bytes.length; i++) {
- byte vb = bytes[i];
- if (vb != RowConstants.ROWKEY_PLACE_HOLDER_BYTE) {
- stripBytes[index++] = vb;
- }
- }
- return stripBytes;
- }
-
- public Dictionary<String> getDictionary(TblColRef col) {
- if (forceNoDict)
- return null;
-
- return (Dictionary<String>) CubeManager.getInstance(seg.getCubeInstance().getConfig()).getDictionary(seg, col);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnOrder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnOrder.java b/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnOrder.java
deleted file mode 100644
index db598c8..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyColumnOrder.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.kv;
-
-import java.util.Collection;
-import java.util.Comparator;
-
-import com.kylinolap.metadata.model.schema.DataType;
-
-/**
- * @author yangli9
- */
-abstract public class RowKeyColumnOrder implements Comparator<String> {
-
- public static final NumberOrder NUMBER_ORDER = new NumberOrder();
- public static final StringOrder STRING_ORDER = new StringOrder();
-
- public static RowKeyColumnOrder getInstance(DataType type) {
- if (type.isNumberFamily())
- return NUMBER_ORDER;
- else
- return STRING_ORDER;
- }
-
- public String max(Collection<String> values) {
- String max = null;
- for (String v : values) {
- if (max == null || compare(max, v) < 0)
- max = v;
- }
- return max;
- }
-
- public String min(Collection<String> values) {
- String min = null;
- for (String v : values) {
- if (min == null || compare(min, v) > 0)
- min = v;
- }
- return min;
- }
-
- public String min(String v1, String v2) {
- if (v1 == null)
- return v2;
- else if (v2 == null)
- return v1;
- else
- return compare(v1, v2) <= 0 ? v1 : v2;
- }
-
- public String max(String v1, String v2) {
- if (v1 == null)
- return v2;
- else if (v2 == null)
- return v1;
- else
- return compare(v1, v2) >= 0 ? v1 : v2;
- }
-
- @Override
- public int compare(String o1, String o2) {
- // consider null
- if (o1 == o2)
- return 0;
- if (o1 == null)
- return -1;
- if (o2 == null)
- return 1;
-
- return compareNonNull(o1, o2);
- }
-
- abstract int compareNonNull(String o1, String o2);
-
- private static class StringOrder extends RowKeyColumnOrder {
- @Override
- public int compareNonNull(String o1, String o2) {
- return o1.compareTo(o2);
- }
- }
-
- private static class NumberOrder extends RowKeyColumnOrder {
- @Override
- public int compareNonNull(String o1, String o2) {
- double d1 = Double.parseDouble(o1);
- double d2 = Double.parseDouble(o2);
- return Double.compare(d1, d2);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/RowKeyDecoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyDecoder.java b/cube/src/main/java/com/kylinolap/cube/kv/RowKeyDecoder.java
deleted file mode 100644
index 527f546..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyDecoder.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.common.RowKeySplitter;
-import com.kylinolap.cube.common.SplittedBytes;
-import com.kylinolap.cube.cuboid.Cuboid;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.model.cube.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public class RowKeyDecoder {
-
- private final CubeDesc cubeDesc;
- private final RowKeyColumnIO colIO;
- private final RowKeySplitter rowKeySplitter;
-
- private Cuboid cuboid;
- private List<String> names;
- private List<String> values;
-
- public RowKeyDecoder(CubeSegment cubeSegment) {
- this.cubeDesc = cubeSegment.getCubeDesc();
- this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 255);
- this.colIO = new RowKeyColumnIO(cubeSegment);
- this.values = new ArrayList<String>();
- }
-
- public long decode(byte[] bytes) throws IOException {
- this.values.clear();
-
- long cuboidId = rowKeySplitter.split(bytes, bytes.length);
- initCuboid(cuboidId);
-
- SplittedBytes[] splits = rowKeySplitter.getSplitBuffers();
-
- int offset = 1; // skip cuboid id part
-
- for (int i = 0; i < this.cuboid.getColumns().size(); i++) {
- TblColRef col = this.cuboid.getColumns().get(i);
- collectValue(col, splits[offset].value, splits[offset].length);
- offset++;
- }
-
- return cuboidId;
- }
-
- private void initCuboid(long cuboidID) {
- if (this.cuboid != null && this.cuboid.getId() == cuboidID) {
- return;
- }
- this.cuboid = Cuboid.findById(cubeDesc, cuboidID);
- }
-
- private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException {
- String strValue = colIO.readColumnString(col, valueBytes, length);
- values.add(strValue);
- }
-
- public RowKeySplitter getRowKeySplitter() {
- return rowKeySplitter;
- }
-
- public void setCuboid(Cuboid cuboid) {
- this.cuboid = cuboid;
- this.names = null;
- }
-
- public List<String> getNames(Map<TblColRef, String> aliasMap) {
- if (names == null) {
- names = buildNameList(aliasMap);
- }
- return names;
- }
-
- private List<String> buildNameList(Map<TblColRef, String> aliasMap) {
- List<TblColRef> columnList = getColumns();
- List<String> result = new ArrayList<String>(columnList.size());
- for (TblColRef col : columnList)
- result.add(findName(col, aliasMap));
- return result;
- }
-
- private String findName(TblColRef column, Map<TblColRef, String> aliasMap) {
- String name = null;
- if (aliasMap != null) {
- name = aliasMap.get(column);
- }
- if (name == null) {
- name = column.getName();
- }
- return name;
- }
-
- public List<TblColRef> getColumns() {
- return cuboid.getColumns();
- }
-
- public List<String> getValues() {
- return values;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append(cuboid.getId());
- for (Object value : values) {
- buf.append(",");
- buf.append(value);
- }
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyEncoder.java b/cube/src/main/java/com/kylinolap/cube/kv/RowKeyEncoder.java
deleted file mode 100644
index 6485d9f..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/RowKeyEncoder.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.cuboid.Cuboid;
-import com.kylinolap.metadata.model.cube.TblColRef;
-
-/**
- * @author George Song (ysong1)
- */
-public class RowKeyEncoder extends AbstractRowKeyEncoder {
-
- private int bytesLength;
- protected int headerLength;
- private RowKeyColumnIO colIO;
-
- protected RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
- super(cuboid);
- colIO = new RowKeyColumnIO(cubeSeg);
- bytesLength = headerLength = RowConstants.ROWKEY_CUBOIDID_LEN; // header
- for (TblColRef column : cuboid.getColumns()) {
- bytesLength += colIO.getColumnLength(column);
- }
- }
-
- public RowKeyColumnIO getColumnIO() {
- return colIO;
- }
-
- public int getColumnOffset(TblColRef col) {
- int offset = RowConstants.ROWKEY_CUBOIDID_LEN;
-
- for (TblColRef dimCol : cuboid.getColumns()) {
- if (col.equals(dimCol))
- return offset;
- offset += colIO.getColumnLength(dimCol);
- }
-
- throw new IllegalArgumentException("Column " + col + " not found on cuboid " + cuboid);
- }
-
- public int getColumnLength(TblColRef col) {
- return colIO.getColumnLength(col);
- }
-
- public int getRowKeyLength() {
- return bytesLength;
- }
-
- public int getHeaderLength() {
- return headerLength;
- }
-
- @Override
- public byte[] encode(Map<TblColRef, String> valueMap) {
- List<byte[]> valueList = new ArrayList<byte[]>();
- for (TblColRef bdCol : cuboid.getColumns()) {
- String value = valueMap.get(bdCol);
- valueList.add(valueStringToBytes(value));
- }
- byte[][] values = valueList.toArray(RowConstants.BYTE_ARR_MARKER);
- return encode(values);
- }
-
- public byte[] valueStringToBytes(String value) {
- if (value == null)
- return null;
- else
- return Bytes.toBytes(value);
- }
-
- @Override
- public byte[] encode(byte[][] values) {
- byte[] bytes = new byte[this.bytesLength];
- int offset = fillHeader(bytes, values);
-
- for (int i = 0; i < cuboid.getColumns().size(); i++) {
- TblColRef column = cuboid.getColumns().get(i);
- int colLength = colIO.getColumnLength(column);
- byte[] value = values[i];
- if (value == null) {
- fillColumnValue(column, colLength, null, 0, bytes, offset);
- } else {
- fillColumnValue(column, colLength, value, value.length, bytes, offset);
- }
- offset += colLength;
-
- }
- return bytes;
- }
-
- protected int fillHeader(byte[] bytes, byte[][] values) {
- int offset = 0;
- System.arraycopy(cuboid.getBytes(), 0, bytes, offset, RowConstants.ROWKEY_CUBOIDID_LEN);
- offset += RowConstants.ROWKEY_CUBOIDID_LEN;
- if (this.headerLength != offset) {
- throw new IllegalStateException("Expected header length is " + headerLength + ". But the offset is " + offset);
- }
- return offset;
- }
-
- protected void fillColumnValue(TblColRef column, int columnLen, byte[] value, int valueLen, byte[] outputValue, int outputValueOffset) {
- // special null value case
- if (value == null) {
- byte[] valueBytes = defaultValue(columnLen);
- System.arraycopy(valueBytes, 0, outputValue, outputValueOffset, columnLen);
- return;
- }
-
- colIO.writeColumn(column, value, valueLen, this.blankByte, outputValue, outputValueOffset);
- }
-
- protected byte[] defaultValue(int length) {
- byte[] values = new byte[length];
- Arrays.fill(values, this.blankByte);
- return values;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/kv/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/kv/RowValueDecoder.java b/cube/src/main/java/com/kylinolap/cube/kv/RowValueDecoder.java
deleted file mode 100644
index c36bc81..0000000
--- a/cube/src/main/java/com/kylinolap/cube/kv/RowValueDecoder.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.cube.kv;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import com.kylinolap.cube.measure.MeasureCodec;
-import com.kylinolap.metadata.model.cube.FunctionDesc;
-import com.kylinolap.metadata.model.cube.HBaseColumnDesc;
-import com.kylinolap.metadata.model.cube.MeasureDesc;
-
-/**
- *
- * @author xjiang
- *
- */
-public class RowValueDecoder implements Cloneable {
-
- private final HBaseColumnDesc hbaseColumn;
- private final MeasureCodec codec;
- private final BitSet projectionIndex;
- private final MeasureDesc[] measures;
- private final List<String> names;
- private Object[] values;
-
- public RowValueDecoder(RowValueDecoder rowValueDecoder) {
- this.hbaseColumn = rowValueDecoder.getHBaseColumn();
- this.projectionIndex = rowValueDecoder.getProjectionIndex();
- this.names = new ArrayList<String>();
- this.measures = hbaseColumn.getMeasures();
- for (MeasureDesc measure : measures) {
- this.names.add(measure.getFunction().getRewriteFieldName());
- }
- this.codec = new MeasureCodec(measures);
- this.values = new Object[measures.length];
- }
-
- public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
- this.hbaseColumn = hbaseColumn;
- this.projectionIndex = new BitSet();
- this.names = new ArrayList<String>();
- this.measures = hbaseColumn.getMeasures();
- for (MeasureDesc measure : measures) {
- this.names.add(measure.getFunction().getRewriteFieldName());
- }
- this.codec = new MeasureCodec(measures);
- this.values = new Object[measures.length];
- }
-
- public void decode(byte[] bytes) {
- codec.decode(ByteBuffer.wrap(bytes), values);
- convertToJavaObjects(values, values);
- }
-
- private void convertToJavaObjects(Object[] mapredObjs, Object[] results) {
- for (int i = 0; i < mapredObjs.length; i++) {
- Object o = mapredObjs[i];
-
- if (o instanceof LongWritable)
- o = Long.valueOf(((LongWritable) o).get());
- else if (o instanceof IntWritable)
- o = Integer.valueOf(((IntWritable) o).get());
- else if (o instanceof DoubleWritable)
- o = Double.valueOf(((DoubleWritable) o).get());
- else if (o instanceof FloatWritable)
- o = Float.valueOf(((FloatWritable) o).get());
-
- results[i] = o;
- }
- }
-
- public void setIndex(int bitIndex) {
- projectionIndex.set(bitIndex);
- }
-
- public HBaseColumnDesc getHBaseColumn() {
- return hbaseColumn;
- }
-
- public BitSet getProjectionIndex() {
- return projectionIndex;
- }
-
- public Object[] getValues() {
- return values;
- }
-
- public List<String> getNames() {
- return names;
- }
-
- public MeasureDesc[] getMeasures() {
- return measures;
- }
-
- public boolean hasMemHungryCountDistinct() {
- for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
- FunctionDesc func = measures[i].getFunction();
- if (func.isCountDistinct() && !func.isHolisticCountDistinct()) {
- return true;
- }
- }
- return false;
- }
-
- public static boolean hasMemHungryCountDistinct(Collection<RowValueDecoder> rowValueDecoders) {
- for (RowValueDecoder decoder : rowValueDecoders) {
- if (decoder.hasMemHungryCountDistinct())
- return true;
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMaxAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMaxAggregator.java
deleted file mode 100644
index 4316ac0..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMaxAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.math.BigDecimal;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) < 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytes() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMinAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMinAggregator.java
deleted file mode 100644
index 52dbcd7..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalMinAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.math.BigDecimal;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) > 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytes() {
- return guessBigDecimalMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSerializer.java b/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSerializer.java
deleted file mode 100644
index 5d893c4..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSerializer.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.kylinolap.common.util.BytesUtil;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSerializer extends MeasureSerializer<BigDecimal> {
-
- @Override
- public void serialize(BigDecimal value, ByteBuffer out) {
- byte[] bytes = value.unscaledValue().toByteArray();
-
- BytesUtil.writeVInt(value.scale(), out);
- BytesUtil.writeVInt(bytes.length, out);
- out.put(bytes);
- }
-
- @Override
- public BigDecimal deserialize(ByteBuffer in) {
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
-
- byte[] bytes = new byte[n];
- in.get(bytes);
-
- return new BigDecimal(new BigInteger(bytes), scale);
- }
-
- @Override
- public BigDecimal valueOf(byte[] value) {
- if (value == null)
- return new BigDecimal(0);
- else
- return new BigDecimal(Bytes.toString(value));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSumAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSumAggregator.java
deleted file mode 100644
index 20bd093..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/BigDecimalSumAggregator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.math.BigDecimal;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal sum = new BigDecimal(0);
-
- @Override
- public void reset() {
- sum = new BigDecimal(0);
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- sum = sum.add(value);
- }
-
- @Override
- public BigDecimal getState() {
- return sum;
- }
-
- @Override
- public int getMemBytes() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/DoubleMaxAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/DoubleMaxAggregator.java
deleted file mode 100644
index 8ed20c8..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/DoubleMaxAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.DoubleWritable;
-
-/**
- * @author yangli9
- *
- */
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleWritable> {
-
- DoubleWritable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(DoubleWritable value) {
- if (max == null)
- max = new DoubleWritable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public DoubleWritable getState() {
- return max;
- }
-
- @Override
- public int getMemBytes() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/DoubleMinAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/DoubleMinAggregator.java
deleted file mode 100644
index e03e386..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/DoubleMinAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.DoubleWritable;
-
-/**
- * @author yangli9
- *
- */
-public class DoubleMinAggregator extends MeasureAggregator<DoubleWritable> {
-
- DoubleWritable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(DoubleWritable value) {
- if (min == null)
- min = new DoubleWritable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public DoubleWritable getState() {
- return min;
- }
-
- @Override
- public int getMemBytes() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/DoubleSerializer.java b/cube/src/main/java/com/kylinolap/cube/measure/DoubleSerializer.java
deleted file mode 100644
index 54e8c1f..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/DoubleSerializer.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.DoubleWritable;
-
-/**
- * @author yangli9
- *
- */
-public class DoubleSerializer extends MeasureSerializer<DoubleWritable> {
-
- // avoid mass object creation
- DoubleWritable current = new DoubleWritable();
-
- @Override
- public void serialize(DoubleWritable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- @Override
- public DoubleWritable deserialize(ByteBuffer in) {
- current.set(in.getDouble());
- return current;
- }
-
- @Override
- public DoubleWritable valueOf(byte[] value) {
- if (value == null)
- current.set(0d);
- else
- current.set(Double.parseDouble(Bytes.toString(value)));
- return current;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/DoubleSumAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/DoubleSumAggregator.java
deleted file mode 100644
index 88b90c7..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/DoubleSumAggregator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.DoubleWritable;
-
-/**
- * @author yangli9
- *
- */
-public class DoubleSumAggregator extends MeasureAggregator<DoubleWritable> {
-
- DoubleWritable sum = new DoubleWritable();
-
- @Override
- public void reset() {
- sum.set(0.0);
- }
-
- @Override
- public void aggregate(DoubleWritable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public DoubleWritable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytes() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/HLLCAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/HLLCAggregator.java
deleted file mode 100644
index cb070d8..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/HLLCAggregator.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
-
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import com.kylinolap.common.hll.HyperLogLogPlusCounter;
-
-/**
- * @author yangli9
- *
- */
-public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
-
- HyperLogLogPlusCounter sum = null;
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(HyperLogLogPlusCounter value) {
- if (sum == null)
- sum = new HyperLogLogPlusCounter(value);
- else
- sum.merge(value);
- }
-
- @Override
- public HyperLogLogPlusCounter getState() {
- return sum;
- }
-
- @Override
- public int getMemBytes() {
- if (sum == null)
- return Integer.MIN_VALUE;
- else
- return 4 + sum.getMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/HLLCSerializer.java b/cube/src/main/java/com/kylinolap/cube/measure/HLLCSerializer.java
deleted file mode 100644
index ef3818b..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/HLLCSerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import com.kylinolap.common.hll.HyperLogLogPlusCounter;
-
-/**
- * @author yangli9
- *
- */
-public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
-
- HyperLogLogPlusCounter current;
-
- HLLCSerializer(int p) {
- current = new HyperLogLogPlusCounter(p);
- }
-
- @Override
- public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
- try {
- value.writeRegisters(out);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
- try {
- current.readRegisters(in);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return current;
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(byte[] value) {
- current.clear();
- if (value == null)
- current.add("__nUlL__");
- else
- current.add(value);
- return current;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/LDCAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/LDCAggregator.java
deleted file mode 100644
index 94f5f19..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/LDCAggregator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Long Distinct Count
- *
- * @author xjiang
- *
- */
-public class LDCAggregator extends MeasureAggregator<LongWritable> {
-
- private static LongWritable ZERO = new LongWritable(0);
-
- private HLLCAggregator hllAgg = null;
- private LongWritable state = new LongWritable(0);
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- this.hllAgg = (HLLCAggregator) agg;
- }
-
- @Override
- public void reset() {
- }
-
- @Override
- public void aggregate(LongWritable value) {
- }
-
- @Override
- public LongWritable getState() {
- if (hllAgg == null) {
- return ZERO;
- } else {
- state.set(hllAgg.getState().getCountEstimate());
- return state;
- }
- }
-
- @Override
- public int getMemBytes() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/LongMaxAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/LongMaxAggregator.java
deleted file mode 100644
index b0b27ca..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/LongMaxAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * @author yangli9
- *
- */
-public class LongMaxAggregator extends MeasureAggregator<LongWritable> {
-
- LongWritable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(LongWritable value) {
- if (max == null)
- max = new LongWritable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public LongWritable getState() {
- return max;
- }
-
- @Override
- public int getMemBytes() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/LongMinAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/LongMinAggregator.java
deleted file mode 100644
index 3d3bc22..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/LongMinAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * @author yangli9
- *
- */
-public class LongMinAggregator extends MeasureAggregator<LongWritable> {
-
- LongWritable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(LongWritable value) {
- if (min == null)
- min = new LongWritable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public LongWritable getState() {
- return min;
- }
-
- @Override
- public int getMemBytes() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/LongSerializer.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/LongSerializer.java b/cube/src/main/java/com/kylinolap/cube/measure/LongSerializer.java
deleted file mode 100644
index 40ff496..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/LongSerializer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.LongWritable;
-
-import com.kylinolap.common.util.BytesUtil;
-
-/**
- * @author yangli9
- *
- */
-public class LongSerializer extends MeasureSerializer<LongWritable> {
-
- // avoid mass object creation
- LongWritable current = new LongWritable();
-
- @Override
- public void serialize(LongWritable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- @Override
- public LongWritable deserialize(ByteBuffer in) {
- current.set(BytesUtil.readVLong(in));
- return current;
- }
-
- @Override
- public LongWritable valueOf(byte[] value) {
- if (value == null)
- current.set(0L);
- else
- current.set(Long.parseLong(Bytes.toString(value)));
- return current;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/LongSumAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/LongSumAggregator.java
deleted file mode 100644
index a62b456..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/LongSumAggregator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * @author yangli9
- *
- */
-public class LongSumAggregator extends MeasureAggregator<LongWritable> {
-
- LongWritable sum = new LongWritable();
-
- @Override
- public void reset() {
- sum.set(0);
- }
-
- @Override
- public void aggregate(LongWritable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public LongWritable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytes() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregator.java b/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregator.java
deleted file mode 100644
index 726cfca..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import com.kylinolap.metadata.model.cube.FunctionDesc;
-import com.kylinolap.metadata.model.schema.DataType;
-
-/**
- * @author yangli9
- *
- */
-abstract public class MeasureAggregator<V> {
-
- public static MeasureAggregator<?> create(String funcName, String returnType) {
- if (FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName) || FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongSumAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalSumAggregator();
- else if (isDouble(returnType))
- return new DoubleSumAggregator();
- } else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) {
- if (DataType.getInstance(returnType).isHLLC())
- return new HLLCAggregator();
- else
- return new LDCAggregator();
- } else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMaxAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMaxAggregator();
- else if (isDouble(returnType))
- return new DoubleMaxAggregator();
- } else if (FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMinAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMinAggregator();
- else if (isDouble(returnType))
- return new DoubleMinAggregator();
- }
- throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + returnType + "'");
- }
-
- public static boolean isBigDecimal(String type) {
- return "decimal".equalsIgnoreCase(type);
- }
-
- public static boolean isDouble(String type) {
- return "double".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "real".equalsIgnoreCase(type);
- }
-
- public static boolean isInteger(String type) {
- return "long".equalsIgnoreCase(type) || "bigint".equalsIgnoreCase(type) || "int".equalsIgnoreCase(type) || "integer".equalsIgnoreCase(type);
- }
-
- public static int guessBigDecimalMemBytes() {
- return 4 // ref
- + 20; // guess of BigDecimal
- }
-
- public static int guessDoubleMemBytes() {
- return 4 // ref
- + 8;
- }
-
- public static int guessLongMemBytes() {
- return 4 // ref
- + 8;
- }
-
- // ============================================================================
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- }
-
- abstract public void reset();
-
- abstract public void aggregate(V value);
-
- abstract public V getState();
-
- // get an estimate of memory consumption
- abstract public int getMemBytes();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregators.java b/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregators.java
deleted file mode 100644
index aa463b8..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/MeasureAggregators.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.kylinolap.metadata.model.cube.FunctionDesc;
-import com.kylinolap.metadata.model.cube.MeasureDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MeasureAggregators {
-
- private MeasureDesc[] descs;
- private MeasureAggregator[] aggs;
-
- public MeasureAggregators(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureAggregators(MeasureDesc... measureDescs) {
- descs = measureDescs;
- aggs = new MeasureAggregator[descs.length];
-
- Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
- for (int i = 0; i < descs.length; i++) {
- FunctionDesc func = descs[i].getFunction();
- aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
- measureIndexMap.put(descs[i].getName(), i);
- }
- // fill back dependent aggregator
- for (int i = 0; i < descs.length; i++) {
- String depMsrRef = descs[i].getDependentMeasureRef();
- if (depMsrRef != null) {
- int index = measureIndexMap.get(depMsrRef);
- aggs[i].setDependentAggregator(aggs[index]);
- }
- }
- }
-
- public void reset() {
- for (int i = 0; i < aggs.length; i++) {
- aggs[i].reset();
- }
- }
-
- public void aggregate(Object[] values) {
- assert values.length == descs.length;
-
- for (int i = 0; i < descs.length; i++) {
- aggs[i].aggregate(values[i]);
- }
- }
-
- public void collectStates(Object[] states) {
- for (int i = 0; i < descs.length; i++) {
- states[i] = aggs[i].getState();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/MeasureCodec.java b/cube/src/main/java/com/kylinolap/cube/measure/MeasureCodec.java
deleted file mode 100644
index 4a511e7..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/MeasureCodec.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.hadoop.io.Text;
-
-import com.kylinolap.metadata.model.cube.MeasureDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MeasureCodec {
-
- int nMeasures;
- MeasureSerializer[] serializers;
-
- public MeasureCodec(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureCodec(MeasureDesc... measureDescs) {
- String[] dataTypes = new String[measureDescs.length];
- for (int i = 0; i < dataTypes.length; i++) {
- dataTypes[i] = measureDescs[i].getFunction().getReturnType();
- }
- init(dataTypes);
- }
-
- public MeasureCodec(String... dataTypes) {
- init(dataTypes);
- }
-
- private void init(String[] dataTypes) {
- nMeasures = dataTypes.length;
- serializers = new MeasureSerializer[nMeasures];
-
- for (int i = 0; i < nMeasures; i++) {
- serializers[i] = MeasureSerializer.create(dataTypes[i]);
- }
- }
-
- public MeasureSerializer getSerializer(int idx) {
- return serializers[idx];
- }
-
- public void decode(Text bytes, Object[] result) {
- decode(ByteBuffer.wrap(bytes.getBytes(), 0, bytes.getLength()), result);
- }
-
- public void decode(ByteBuffer buf, Object[] result) {
- assert result.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- result[i] = serializers[i].deserialize(buf);
- }
- }
-
- public void encode(Object[] values, ByteBuffer out) {
- assert values.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- serializers[i].serialize(values[i], out);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/MeasureSerializer.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/MeasureSerializer.java b/cube/src/main/java/com/kylinolap/cube/measure/MeasureSerializer.java
deleted file mode 100644
index 9b2ece7..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/MeasureSerializer.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.cube.measure;
-
-import java.util.HashMap;
-
-import com.kylinolap.common.util.BytesSerializer;
-import com.kylinolap.metadata.model.schema.DataType;
-
-/**
- * @author yangli9
- *
- */
-abstract public class MeasureSerializer<T> implements BytesSerializer<T> {
-
- final static HashMap<String, Class<?>> implementations = new HashMap<String, Class<?>>();
- static {
- implementations.put("decimal", BigDecimalSerializer.class);
- implementations.put("double", DoubleSerializer.class);
- implementations.put("float", DoubleSerializer.class);
- implementations.put("bigint", LongSerializer.class);
- implementations.put("long", LongSerializer.class);
- implementations.put("integer", LongSerializer.class);
- implementations.put("int", LongSerializer.class);
- }
-
- public static MeasureSerializer<?> create(String dataType) {
- DataType type = DataType.getInstance(dataType);
- if (type.isHLLC()) {
- return new HLLCSerializer(type.getPrecision());
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + dataType);
-
- try {
- return (MeasureSerializer<?>) clz.newInstance();
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- abstract public T valueOf(byte[] value);
-
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedLenMeasureCodec.java b/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedLenMeasureCodec.java
deleted file mode 100644
index e8a7a08..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedLenMeasureCodec.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package com.kylinolap.cube.measure.fixedlen;
-
-import com.kylinolap.metadata.model.schema.DataType;
-
-abstract public class FixedLenMeasureCodec<T> {
-
- public static FixedLenMeasureCodec<?> get(DataType type) {
- return new FixedPointLongCodec(type.getScale());
- }
-
- abstract public int getLength();
-
- abstract public T valueOf(String value);
-
- abstract public String toString(T value);
-
- abstract public T read(byte[] buf, int offset);
-
- abstract public void write(T v, byte[] buf, int offset);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedPointLongCodec.java b/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedPointLongCodec.java
deleted file mode 100644
index 84ba9fb..0000000
--- a/cube/src/main/java/com/kylinolap/cube/measure/fixedlen/FixedPointLongCodec.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package com.kylinolap.cube.measure.fixedlen;
-
-import org.apache.hadoop.io.LongWritable;
-
-import com.kylinolap.common.util.BytesUtil;
-
-public class FixedPointLongCodec extends FixedLenMeasureCodec<LongWritable> {
-
- private static final int SIZE = 8;
- // number of digits after decimal point
- int scale;
- double scalePower;
- // avoid mass object creation
- LongWritable current = new LongWritable();
-
- public FixedPointLongCodec(int scale) {
- scale = Math.max(0, scale);
- this.scale = scale;
- this.scalePower = Math.pow(10, scale);
- }
-
- @Override
- public int getLength() {
- return SIZE;
- }
-
- @Override
- public LongWritable valueOf(String value) {
- if (value == null)
- current.set(0L);
- else
- current.set((long) (Double.parseDouble(value) * scalePower));
- return current;
- }
-
- @Override
- public String toString(LongWritable value) {
- if (scale == 0)
- return value.toString();
- else
- return "" + (value.get() / scalePower);
- }
-
- @Override
- public LongWritable read(byte[] buf, int offset) {
- current.set(BytesUtil.readLong(buf, offset, SIZE));
- return current;
- }
-
- @Override
- public void write(LongWritable v, byte[] buf, int offset) {
- BytesUtil.writeLong(v.get(), buf, offset, SIZE);
- }
-}