You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/25 12:12:48 UTC
[16/50] [abbrv] kylin git commit: KYLIN-1858 remove all ii related
code
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
deleted file mode 100644
index dfaf106..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionarySerializer;
-import org.apache.kylin.invertedindex.index.ColumnValueContainer;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.datatype.DataType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- */
-public class IIKeyValueCodec implements KeyValueCodec {
-
- public static final int SHARD_LEN = 2;
- public static final int TIMEPART_LEN = 8;
- public static final int COLNO_LEN = 2;
- protected final TableRecordInfoDigest digest;
-
- public IIKeyValueCodec(TableRecordInfoDigest digest) {
- this.digest = digest;
- }
-
- @Override
- public Collection<IIRow> encodeKeyValue(Slice slice) {
- ArrayList<IIRow> result = Lists.newArrayList();
- ColumnValueContainer[] containers = slice.getColumnValueContainers();
- for (int col = 0; col < containers.length; col++) {
- if (containers[col] instanceof CompressedValueContainer) {
- final IIRow row = collectKeyValues(slice, col, (CompressedValueContainer) containers[col]);
- result.add(row);
- } else {
- throw new IllegalArgumentException("Unknown container class " + containers[col].getClass());
- }
- }
- return result;
- }
-
- private IIRow collectKeyValues(Slice slice, int col, CompressedValueContainer container) {
- ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col);
- ImmutableBytesWritable value = container.toBytes();
- final Dictionary<?> dictionary = slice.getLocalDictionaries() != null ? slice.getLocalDictionaries()[col] : null;
- if (dictionary == null) {
- return new IIRow(key, value, new ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY));
- } else {
- ByteArray bytes = DictionarySerializer.serialize(dictionary);
- return new IIRow(key, value, new ImmutableBytesWritable(bytes.array(), bytes.offset(), bytes.length()));
- }
- }
-
- ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) {
- byte[] bytes = new byte[20];
- int len = encodeKey(shard, timestamp, col, bytes, 0);
- return new ImmutableBytesWritable(bytes, 0, len);
- }
-
- int encodeKey(short shard, long timestamp, int col, byte[] buf, int offset) {
- int i = offset;
-
- BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN);
- i += SHARD_LEN;
- BytesUtil.writeLong(timestamp, buf, i, TIMEPART_LEN);
- i += TIMEPART_LEN;
-
- BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN);
- i += COLNO_LEN;
-
- return i - offset;
- }
-
- @Override
- public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
- return new IIRowDecoder(digest, kvs.iterator());
- //return new Decoder(kvs, incompleteDigest);
- }
-
- private static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Dictionary<?>[] dictionaries) {
- int[] dictMaxIds = new int[nColumns];
- int[] lengths = new int[nColumns];
- final boolean emptyDictionary = Array.isEmpty(dictionaries);
- for (int i = 0; i < nColumns; ++i) {
- if (isMetric[i]) {
- final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getType(dataTypes[i]));
- lengths[i] = fixedLenMeasureCodec.getLength();
- } else {
- if (emptyDictionary) {
- final DataType dataType = DataType.getType(dataTypes[i]);
- if (dataType.isNumberFamily()) {
- lengths[i] = 16;
- } else if (dataType.isStringFamily()) {
- lengths[i] = 256;
- } else if (dataType.isDateTimeFamily()) {
- lengths[i] = 19;
- } else {
- throw new RuntimeException("invalid data type:" + dataType);
- }
- dictMaxIds[i] = Integer.MAX_VALUE;
- } else {
- final Dictionary<?> dictionary = dictionaries[i];
- lengths[i] = dictionary.getSizeOfId();
- dictMaxIds[i] = dictionary.getMaxId();
- }
- }
- }
- // offsets
- int pos = 0;
- int[] offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += lengths[i];
- }
-
- int byteFormLen = pos;
-
- return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes);
- }
-
- protected static class IIRowDecoder implements Iterable<Slice> {
-
- protected final TableRecordInfoDigest incompleteDigest;
- protected final Iterator<IIRow> iiRowIterator;
- protected Iterator<IIRow> feedingIterator;//this is for extending
-
- protected IIRowDecoder(TableRecordInfoDigest digest, Iterator<IIRow> iiRowIterator) {
- this.incompleteDigest = digest;
- this.iiRowIterator = iiRowIterator;
- this.feedingIterator = this.iiRowIterator;
- }
-
- @Override
- public Iterator<Slice> iterator() {
- return new Iterator<Slice>() {
- @Override
- public boolean hasNext() {
- return iiRowIterator.hasNext();
- }
-
- @Override
- public Slice next() {
- int columns = 0;
- ColumnValueContainer[] valueContainers = new ColumnValueContainer[incompleteDigest.getColumnCount()];
- Dictionary<?>[] localDictionaries = new Dictionary<?>[incompleteDigest.getColumnCount()];
- boolean firstTime = true;
- short curShard = 0;
- long curTimestamp = 0;
- short lastShard = 0;
- long lastTimestamp = 0;
-
- while (feedingIterator.hasNext() && columns < incompleteDigest.getColumnCount()) {
- final IIRow row = feedingIterator.next();
- final ImmutableBytesWritable key = row.getKey();
- int i = key.getOffset();
- curShard = (short) BytesUtil.readUnsigned(key.get(), i, SHARD_LEN);
- i += SHARD_LEN;
- curTimestamp = BytesUtil.readLong(key.get(), i, TIMEPART_LEN);
- i += TIMEPART_LEN;
-
- if (!firstTime) {
- Preconditions.checkArgument(curShard == lastShard, "shard should be equals in one slice, curShard is" + curShard + " lastShard is " + lastShard);
- Preconditions.checkArgument(curTimestamp == lastTimestamp, "timestamp should be equals in one slice, curTimestamp is" + curTimestamp + " lastTimestamp is " + lastTimestamp);
- }
-
- int curCol = BytesUtil.readUnsigned(key.get(), i, COLNO_LEN);
- if (incompleteDigest.isMetrics(curCol)) {
- CompressedValueContainer c = new CompressedValueContainer(incompleteDigest, curCol, 0);
- c.fromBytes(row.getValue());
- valueContainers[curCol] = c;
- } else {
- final ImmutableBytesWritable dictBytes = row.getDictionary();
- if (dictBytes.getLength() != 0) {
- final Dictionary<?> dictionary = DictionarySerializer.deserialize(new ByteArray(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength()));
- CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), dictionary.getMaxId() - dictionary.getMinId() + 1, 0);
- c.fromBytes(row.getValue());
- valueContainers[curCol] = c;
- localDictionaries[curCol] = dictionary;
- } else {
- CompressedValueContainer c = new CompressedValueContainer(incompleteDigest.length(curCol), incompleteDigest.getMaxID(curCol) - 0 + 1, 0);
- c.fromBytes(row.getValue());
- valueContainers[curCol] = c;
- }
- }
- columns++;
- lastShard = curShard;
- lastTimestamp = curTimestamp;
- firstTime = false;
- }
- Preconditions.checkArgument(columns == incompleteDigest.getColumnCount(), "column count is " + columns + " should be equals to incompleteDigest.getColumnCount() " + incompleteDigest.getColumnCount());
-
- TableRecordInfoDigest digest = createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries);
- Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers);
- slice.setLocalDictionaries(localDictionaries);
- return slice;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
deleted file mode 100644
index 8747916..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.FIFOIterable;
-import org.apache.kylin.common.util.FIFOIterator;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class IIKeyValueCodecWithState extends IIKeyValueCodec {
-
- public IIKeyValueCodecWithState(TableRecordInfoDigest digest) {
- super(digest);
- }
-
- /**
- *
- * @param kvs kvs must be a {@link org.apache.kylin.common.util.FIFOIterable } to avoid {@link java.util.ConcurrentModificationException}.
- * @return
- */
- @Override
- public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) {
- if (!(kvs instanceof FIFOIterable)) {
- throw new IllegalArgumentException("kvs must be a {@link org.apache.kylin.common.util.FIFOIterable } to avoid {@link java.util.ConcurrentModificationException}.");
- }
- return new IIRowDecoderWithState(digest, kvs.iterator());
- }
-
- //TODO refactor this class, does not have to extend IIKeyValueCodec, composite might be enough, mhb
- protected static class IIRowDecoderWithState extends IIRowDecoder {
-
- final LinkedList<IIRow> buffer = Lists.newLinkedList();
- private Iterator<Slice> superIterator = null;
-
- private IIRowDecoderWithState(TableRecordInfoDigest digest, Iterator<IIRow> iiRowIterator) {
- super(digest, iiRowIterator);
- this.feedingIterator = new FIFOIterator<>(buffer);
- }
-
- private Iterator<Slice> getSuperIterator() {
- if (superIterator == null) {
- superIterator = super.iterator();
- }
- return superIterator;
- }
-
- @Override
- public Iterator<Slice> iterator() {
- return new Iterator<Slice>() {
- @Override
- public boolean hasNext() {
- while (buffer.size() < incompleteDigest.getColumnCount() && iiRowIterator.hasNext()) {
- buffer.add(iiRowIterator.next());
- }
- return buffer.size() == incompleteDigest.getColumnCount();
- }
-
- @Override
- public Slice next() {
- while (buffer.size() < incompleteDigest.getColumnCount() && iiRowIterator.hasNext()) {
- buffer.add(iiRowIterator.next());
- }
- Preconditions.checkArgument(buffer.size() == incompleteDigest.getColumnCount(), "not enough IIRows!");
- Slice ret = IIRowDecoderWithState.this.getSuperIterator().next();
- buffer.clear();
- return ret;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
deleted file mode 100644
index 9d13487..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public final class IIRow {
-
- private final ImmutableBytesWritable key;
- private final ImmutableBytesWritable value;
- private final ImmutableBytesWritable dictionary;
-
- public IIRow(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) {
- this.key = key;
- this.value = value;
- this.dictionary = dictionary;
- }
-
- public IIRow() {
- this(new ImmutableBytesWritable(), new ImmutableBytesWritable(), new ImmutableBytesWritable());
- }
-
- public ImmutableBytesWritable getKey() {
- return key;
- }
-
- public ImmutableBytesWritable getValue() {
- return value;
- }
-
- public ImmutableBytesWritable getDictionary() {
- return dictionary;
- }
-
- public void updateWith(Cell c) {
- if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) {
- this.getKey().set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- this.getValue().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) {
- this.getDictionary().set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- }
- }
-
- public List<Cell> makeCells() {
- Cell a = new KeyValue(this.getKey().copyBytes(), IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, this.getValue().copyBytes());
- Cell b = new KeyValue(this.getKey().copyBytes(), IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, this.getDictionary().copyBytes());
- return Lists.newArrayList(a, b);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
deleted file mode 100644
index 7843ee7..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.invertedindex.model;
-
-import java.util.Collection;
-
-import org.apache.kylin.invertedindex.index.Slice;
-
-/**
- */
-public interface KeyValueCodec {
-
- Collection<IIRow> encodeKeyValue(Slice slice);
-
- Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
deleted file mode 100644
index 36a8781..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.invertedindex.util;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import javax.annotation.Nullable;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-
-/**
- */
-public final class IIDictionaryBuilder {
-
- private IIDictionaryBuilder() {
- }
-
- public static Dictionary<?>[] buildDictionary(List<List<String>> table, IIDesc desc) throws IOException {
- HashMultimap<TblColRef, String> valueMap = HashMultimap.create();
- final List<TblColRef> allColumns = desc.listAllColumns();
- for (List<String> row : table) {
- for (int i = 0; i < row.size(); i++) {
- String cell = row.get(i);
- if (!desc.isMetricsCol(i)) {
- valueMap.put(allColumns.get(i), cell);
- }
- }
- }
-
- Dictionary<?>[] result = new Dictionary<?>[allColumns.size()];
- for (TblColRef tblColRef : valueMap.keySet()) {
- final Collection<byte[]> bytes = Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- return input == null ? null : input.getBytes();
- }
- });
- final Dictionary<?> dict = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(bytes));
- result[desc.findColumn(tblColRef)] = dict;
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java
deleted file mode 100644
index 96dcbfc..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class IIDescManagerTest extends LocalFileMetadataTestCase {
-
- public static final String TEST_II_DESC_NAME = "test_kylin_ii_left_join_desc";
-
- @Before
- public void setup() {
- createTestMetadata();
- }
-
- @After
- public void clean() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCRUD() throws IOException {
- IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
-
- String newDescName = "Copy of " + TEST_II_DESC_NAME;
-
- try {
- IIDesc testRecord = mgr.getIIDesc(newDescName);
- if (testRecord != null)
- mgr.removeIIDesc(testRecord);
- } catch (IOException e) {
- // just ensure the old one is removed
- }
-
- Assert.assertNull(mgr.getIIDesc(newDescName));
- IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- desc.setName(newDescName);
- desc.setLastModified(0);
-
- mgr.createIIDesc(desc);
-
- desc = mgr.getIIDesc(newDescName);
-
- Assert.assertNotNull(desc);
-
- mgr.updateIIDesc(desc); // this will trigger cache wipe; please ignore the HTTP error in logs.
-
- mgr.removeIIDesc(desc);
-
- Assert.assertNull(mgr.getIIDesc(newDescName));
-
- }
-
- @Test
- public void testReload() throws IOException {
- IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
-
- IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- // do some modification
- desc.setUuid(UUID.randomUUID().toString());
-
- IIDesc newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- Assert.assertEquals(desc, newDesc);
-
- // reload the cache
- mgr.reloadIIDescLocal(TEST_II_DESC_NAME);
-
- newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- Assert.assertNotEquals(desc, newDesc);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java
deleted file mode 100644
index 4b5517f..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class IIDescTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() {
- this.createTestMetadata();
-
- }
-
- @After
- public void clear() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testGetIIDesc() {
-
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
- DataModelDesc model = iiDesc.getModel();
- Assert.assertNotNull(iiDesc);
- Assert.assertNotNull(model);
-
- }
-
- @Test
- public void testSerialization() throws IOException {
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
- String str = JsonUtil.writeValueAsIndentString(iiDesc);
- System.out.println(str);
- IIDesc desc2 = JsonUtil.readValue(str, IIDesc.class);
-
- Assert.assertEquals(iiDesc, desc2);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
deleted file mode 100644
index 16bbdf0..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class IIInstanceTest extends LocalFileMetadataTestCase {
- @Before
- public void setup() {
- createTestMetadata();
- }
-
- @After
- public void clean() {
- cleanupTestMetadata();
- }
-
- @Test
- public void testGetIIsByDesc() throws IOException {
- IIManager mgr = IIManager.getInstance(getTestConfig());
-
- List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_left_join_desc");
-
- Assert.assertTrue(iiInstances.size() > 0);
-
- }
-
- @Test
- public void testCreateIIInstance() throws IOException {
-
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
-
- IIInstance ii = IIInstance.create("new ii", "default", iiDesc);
-
- IIManager iiMgr = IIManager.getInstance(getTestConfig());
-
- List<IIInstance> allIIList = iiMgr.listAllIIs();
-
- iiMgr.createII(ii);
-
- Assert.assertNotNull(iiMgr.getII("new ii"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
deleted file mode 100644
index 7b6a688..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.ShardingSliceBuilder;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.IIRow;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.SetMultimap;
-import com.google.common.collect.Sets;
-
-public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
-
- IIInstance ii;
- TableRecordInfo info;
- List<String> lines;
- private Dictionary<?>[] dictionaryMap;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
-
- File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv");
- FileInputStream in = new FileInputStream(file);
- this.lines = IOUtils.readLines(in, "UTF-8");
- in.close();
-
- dictionaryMap = buildDictionary(Lists.transform(lines, new Function<String, List<String>>() {
- @Nullable
- @Override
- public List<String> apply(@Nullable String input) {
- return Lists.newArrayList(input.split(","));
- }
- }), ii.getDescriptor());
- this.info = new TableRecordInfo(ii.getDescriptor(), dictionaryMap);
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCompressedValueContainer() {
- // create container
- CompressedValueContainer container = new CompressedValueContainer(info.getDigest(), 0, 500);
- Dictionary<String> dict = info.dict(0);
-
- byte[] buf = new byte[dict.getSizeOfId()];
- ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf);
-
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId());
- container.append(bytes);
- }
- BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId());
- container.append(bytes);
- container.closeForChange();
-
- // copy by serialization
- ImmutableBytesWritable copy = container.toBytes();
- CompressedValueContainer container2 = new CompressedValueContainer(info.getDigest(), 0, 500);
- container2.fromBytes(copy);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(v, value);
- }
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value);
- assertEquals(container, container2);
- }
-
- @Test
- public void testCodec() throws IOException {
- List<TableRecord> records = loadRecordsSorted();
- System.out.println(records.size() + " records");
- List<Slice> slices = buildTimeSlices(records);
- System.out.println(slices.size() + " slices");
-
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- List<IIRow> kvs = encodeKVs(codec, slices);
- System.out.println(kvs.size() + " KV pairs");
-
- List<Slice> slicesCopy = decodeKVs(codec, kvs);
- assertEquals(slices.size(), slicesCopy.size());
- for (int i = 0; i < slices.size(); i++) {
- assertEquals(slices.get(i), slicesCopy.get(i));
- }
-
- List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
- assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy));
- dump(recordsCopy);
- }
-
- private Dictionary<?>[] buildDictionary(List<List<String>> table, IIDesc desc) throws IOException {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
- Set<TblColRef> dimensionColumns = Sets.newHashSet();
- for (int i = 0; i < desc.listAllColumns().size(); i++) {
- if (!desc.isMetricsCol(i)) {
- dimensionColumns.add(desc.listAllColumns().get(i));
- }
- }
- for (List<String> row : table) {
- for (int i = 0; i < row.size(); i++) {
- String cell = row.get(i);
- valueMap.put(desc.listAllColumns().get(i), cell);
- }
- }
- Dictionary<?>[] result = new Dictionary<?>[desc.listAllColumns().size()];
- for (TblColRef tblColRef : valueMap.keys()) {
- result[desc.findColumn(tblColRef)] = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- try {
- return input.getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
- })));
- }
- return result;
- }
-
- private List<TableRecord> loadRecordsSorted() throws IOException {
- List<TableRecord> records = Lists.newArrayList();
- for (String line : lines) {
- String[] fields = line.split(",");
- TableRecord rec = info.createTableRecord();
- for (int col = 0; col < fields.length; col++) {
- rec.setValueString(col, fields[col]);
- }
- records.add(rec);
- }
-
- Collections.sort(records, new Comparator<TableRecord>() {
- @Override
- public int compare(TableRecord a, TableRecord b) {
- long x = a.getTimestamp() - b.getTimestamp();
- if (x > 0)
- return 1;
- else if (x == 0)
- return 0;
- else
- return -1;
- }
- });
-
- return records;
- }
-
- private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException {
- ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
- List<Slice> slices = Lists.newArrayList();
- for (TableRecord rec : records) {
- //here assume there less records than slice size for each shard
- Slice slice = builder.append(rec);
- if (slice != null) {
- slice.setLocalDictionaries(dictionaryMap);
- slices.add(slice);
- }
- }
- List<Slice> finals = builder.close();
- for (Slice slice : finals) {
- slice.setLocalDictionaries(dictionaryMap);
- }
- slices.addAll(finals);
-
- Collections.sort(slices);
- return slices;
- }
-
- private List<IIRow> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) {
-
- List<IIRow> kvs = Lists.newArrayList();
- for (Slice slice : slices) {
- kvs.addAll(codec.encodeKeyValue(slice));
- }
- return kvs;
- }
-
- private List<Slice> decodeKVs(IIKeyValueCodec codec, List<IIRow> kvs) {
- List<Slice> slices = Lists.newArrayList();
- for (Slice slice : codec.decodeKeyValue(kvs)) {
- slices.add(slice);
- }
- return slices;
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (RawTableRecord rec : slice) {
- records.add(new TableRecord((RawTableRecord) rec.clone(), info));
- }
- }
- return records;
- }
-
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- System.out.println(rec.toString());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java
deleted file mode 100644
index 943e76c..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.junit.Test;
-
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-
-/**
- */
-public class LZFTest {
- @Test
- public void test() throws IOException {
-
- byte[] raw = new byte[] { 1, 2, 3, 3, 2, 23 };
- byte[] data = LZFEncoder.encode(raw);
-
- byte[] data2 = new byte[data.length * 2];
- java.lang.System.arraycopy(data, 0, data2, 0, data.length);
- ImmutableBytesWritable bytes = new ImmutableBytesWritable();
- bytes.set(data2, 0, data.length);
-
- try {
- byte[] uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
- } catch (IOException e) {
- throw new RuntimeException("LZF decode failure", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
deleted file mode 100644
index 6b21282..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.measure;
-
-import org.apache.kylin.metadata.datatype.DataType;
-import org.junit.Test;
-
-/**
- */
-public class FixedPointLongCodecTest {
-
- @Test
- public void testEncode1() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12.12345");
- org.junit.Assert.assertEquals(1212345, x);
- }
-
- @Test
- public void testEncode2() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12.1234");
- org.junit.Assert.assertEquals(1212340, x);
- }
-
- @Test
- public void testEncode3() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12.123456");
- org.junit.Assert.assertEquals(1212345, x);
- }
-
- @Test
- public void testEncode4() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12");
- org.junit.Assert.assertEquals(1200000, x);
- }
-
- @Test
- public void testDecode1() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
- String x = codec.restoreDecimalPoint(1212345);
- org.junit.Assert.assertEquals("12.12345", x);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 271bae8..a96dcd9 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -74,12 +74,6 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-linq4j</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index f0a3178..cf18b20 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -42,7 +42,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
@BeforeClass
public static void setUp() throws SQLException {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 2);
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
Candidate.setPriorities(priorities);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
deleted file mode 100644
index fd74dc1..0000000
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.query.routing.Candidate;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runners.Parameterized;
-
-import com.google.common.collect.Maps;
-
-/**
- */
-@Ignore("II query is no longer maintained")
-//@RunWith(Parameterized.class)
-public class ITIIQueryTest extends ITKylinQueryTest {
- @BeforeClass
- public static void setUp() throws Exception {
-
- // give II higher priority than other realizations
- Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 0);
- priorities.put(RealizationType.HYBRID, 1);
- priorities.put(RealizationType.CUBE, 2);
- Candidate.setPriorities(priorities);
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- ITKylinQueryTest.tearDown();//invoke super class
- Candidate.restorePriorities();
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { { "inner" }, { "left" } });
- }
-
- public ITIIQueryTest(String joinType) throws Exception {
-
- ITKylinQueryTest.clean();
-
- ITKylinQueryTest.joinType = joinType;
- ITKylinQueryTest.setupAll();
-
- }
-
- @Test
- public void testSingleRunQuery() throws Exception {
- super.testSingleRunQuery();
- }
-
- @Test
- public void testDetailedQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_ii", null, true);
- }
-
- @Override
- @Test
- @Ignore("Skip Precisely Distinct Count Queries for II")
- public void testPreciselyDistinctCountQuery() {
- }
-
- @Override
- @Test
- @Ignore("Skip Dimension Distinct Count Queries for II")
- public void testDimDistinctCountQuery() {
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index d909a99..a53c624 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -51,7 +51,6 @@ public class ITKylinQueryTest extends KylinTestBase {
@BeforeClass
public static void setUp() throws Exception {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 2);
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
Candidate.setPriorities(priorities);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
deleted file mode 100644
index a04c5ae..0000000
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.ii;
-
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.cube.v1.HBaseClientKVIterator;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- */
-public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
-
- IIInstance ii;
- IISegment seg;
- HConnection hconn;
-
- TableRecordInfo info;
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
-
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
- this.seg = ii.getFirstSegment();
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- hconn = HConnectionManager.createConnection(hconf);
-
- this.info = new TableRecordInfo(seg);
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testLoad() throws Exception {
-
- String tableName = seg.getStorageLocationIdentifier();
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
-
- List<Slice> slices = Lists.newArrayList();
- HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES);
- try {
- for (Slice slice : codec.decodeKeyValue(kvIterator)) {
- slices.add(slice);
- }
- } finally {
- kvIterator.close();
- }
-
- List<TableRecord> records = iterateRecords(slices);
- //dump(records);
- System.out.println("table name:" + tableName + " has " + records.size() + " records");
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (RawTableRecord rec : slice) {
- records.add(new TableRecord((RawTableRecord) rec.clone(), info));
- }
- }
- return records;
- }
-
- @SuppressWarnings("unused")
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- byte[] x = rec.getBytes();
- String y = BytesUtil.toReadableText(x);
- System.out.println(y);
- System.out.println();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2def6d2..7a211fc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -677,7 +677,6 @@
<module>server-base</module>
<module>server</module>
<module>jdbc</module>
- <module>invertedindex</module>
<module>assembly</module>
<module>tool</module>
<module>kylin-it</module>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 87a4eba..a0b8d94 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -48,12 +48,6 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-linq4j</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index 28c60a8..c7c99ab 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -35,7 +35,6 @@ public class Candidate implements Comparable<Candidate> {
static {
DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0);
DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1);
- DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 2);
}
/** for test only */
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 7197f03..5f07adf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -30,8 +30,6 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.engine.streaming.StreamingManager;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.Output;
@@ -94,14 +92,6 @@ public abstract class BasicService {
return ExecutableManager.getInstance(getConfig());
}
- public IIDescManager getIIDescManager() {
- return IIDescManager.getInstance(getConfig());
- }
-
- public IIManager getIIManager() {
- return IIManager.getInstance(getConfig());
- }
-
public BadQueryHistoryManager getBadQueryHistoryManager() {
return BadQueryHistoryManager.getInstance(getConfig());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 9185544..02efe70 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -38,8 +38,6 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.engine.streaming.StreamingManager;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -186,28 +184,16 @@ public class CacheService extends BasicService {
case PROJECT:
reloadProjectCache(cacheKey);
break;
- case INVERTED_INDEX:
- //II update does not need to update storage cache because it is dynamic already
- getIIManager().reloadIILocal(cacheKey);
- getHybridManager().reloadHybridInstanceByChild(RealizationType.INVERTED_INDEX, cacheKey);
- getProjectManager().clearL2Cache();
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().reloadIIDescLocal(cacheKey);
- break;
case TABLE:
getMetadataManager().reloadTableCache(cacheKey);
- IIDescManager.clearCache();
CubeDescManager.clearCache();
break;
case EXTERNAL_FILTER:
getMetadataManager().reloadExtFilter(cacheKey);
- IIDescManager.clearCache();
CubeDescManager.clearCache();
break;
case DATA_MODEL:
getMetadataManager().reloadDataModelDesc(cacheKey);
- IIDescManager.clearCache();
CubeDescManager.clearCache();
break;
case ALL:
@@ -215,8 +201,6 @@ public class CacheService extends BasicService {
MetadataManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
HybridManager.clearCache();
RealizationRegistry.clearCache();
ProjectManager.clearCache();
@@ -259,12 +243,6 @@ public class CacheService extends BasicService {
case PROJECT:
ProjectManager.clearCache();
break;
- case INVERTED_INDEX:
- getIIManager().removeIILocal(cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().removeIIDescLocal(cacheKey);
- break;
case TABLE:
throw new UnsupportedOperationException(log);
case EXTERNAL_FILTER:
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 9d8ccfb..4cfa209 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
@@ -117,14 +116,6 @@ public class ModelService extends BasicService {
}
}
- //check II desc exist
- List<IIDesc> iiDescs = getIIDescManager().listAllDesc();
- for (IIDesc iidesc : iiDescs) {
- if (iidesc.getModelName().equals(desc.getName())) {
- throw new InternalErrorException("Model referenced by IIDesc.");
- }
- }
-
getMetadataManager().dropModel(desc);
accessService.clean(desc, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index f8dc945..ae4c089 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -21,8 +21,6 @@ package org.apache.kylin.rest.service;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationRegistry;
@@ -65,8 +63,6 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
MetadataManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
RealizationRegistry.clearCache();
ProjectManager.clearCache();
CacheService.removeAllOLAPDataSources();
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 943b2d4..0cb42db 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -46,11 +46,6 @@
<artifactId>kylin-engine-streaming</artifactId>
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
<!-- Env & Test -->
<dependency>
@@ -135,7 +130,6 @@
<include>org.apache.kylin:kylin-core-metadata</include>
<include>org.apache.kylin:kylin-core-dictionary</include>
<include>org.apache.kylin:kylin-core-cube</include>
- <include>org.apache.kylin:kylin-invertedindex</include>
<include>com.ning:compress-lzf</include>
<include>org.roaringbitmap:RoaringBitmap</include>
<!-- below for inverted index only -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index 20bc229..e7a322c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -18,12 +18,12 @@
package org.apache.kylin.storage.hbase;
+import com.google.common.base.Preconditions;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.engine.mr.IMROutput;
import org.apache.kylin.engine.mr.IMROutput2;
-import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
@@ -36,8 +36,6 @@ import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
-import com.google.common.base.Preconditions;
-
@SuppressWarnings("unused")
//used by reflection
public class HBaseStorage implements IStorage {
@@ -46,21 +44,10 @@ public class HBaseStorage implements IStorage {
public final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery";
public static String overwriteStorageQuery = null;//for test case
- private final static String defaultIIStorageQuery = "org.apache.kylin.storage.hbase.ii.InvertedIndexStorageQuery";
-
@Override
public IStorageQuery createQuery(IRealization realization) {
- if (realization.getType() == RealizationType.INVERTED_INDEX) {
- IStorageQuery ret;
- try {
- ret = (IStorageQuery) Class.forName(defaultIIStorageQuery).getConstructor(IIInstance.class).newInstance((IIInstance) realization);
- } catch (Exception e) {
- throw new RuntimeException("Failed to initialize storage query for " + defaultIIStorageQuery, e);
- }
- return ret;
-
- } else if (realization.getType() == RealizationType.CUBE) {
+ if (realization.getType() == RealizationType.CUBE) {
CubeInstance cubeInstance = (CubeInstance) realization;
String cubeStorageQuery;
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java
index e142536..65c5f92 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java
@@ -29,7 +29,6 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowKeyEncoder;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -55,20 +54,7 @@ public class CoprocessorProjector {
byte[] mask = rowKeyMaskEncoder.encode(new byte[cuboid.getColumns().size()][]);
return new CoprocessorProjector(mask, dimensionColumns.size() != 0);
}
-
- public static CoprocessorProjector makeForEndpoint(final TableRecordInfo tableInfo, final Collection<TblColRef> groupby) {
- byte[] mask = new byte[tableInfo.getDigest().getByteFormLen()];
- int maskIdx = 0;
- for (int i = 0; i < tableInfo.getDigest().getColumnCount(); ++i) {
- TblColRef tblColRef = tableInfo.getColumns().get(i);
- int length = tableInfo.getDigest().length(i);
- byte bits = groupby.contains(tblColRef) ? (byte) 0xff : 0x00;
- for (int j = 0; j < length; ++j) {
- mask[maskIdx++] = bits;
- }
- }
- return new CoprocessorProjector(mask, groupby.size() != 0);
- }
+
public static byte[] serialize(CoprocessorProjector o) {
ByteBuffer buf = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
index a6e3073..d47bf61 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
@@ -27,7 +27,6 @@ import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -39,17 +38,6 @@ import com.google.common.collect.Maps;
*/
public class CoprocessorRowType {
- //for endpoint
- public static CoprocessorRowType fromTableRecordInfo(TableRecordInfo tableRecordInfo, List<TblColRef> cols) {
-
- int[] colSizes = new int[cols.size()];
- for (int i = 0; i < cols.size(); i++) {
- colSizes[i] = tableRecordInfo.getDigest().length(i);
- }
-
- //TODO:check0
- return new CoprocessorRowType(cols.toArray(new TblColRef[cols.size()]), colSizes, 0);
- }
//for observer
public static CoprocessorRowType fromCuboid(CubeSegment seg, Cuboid cuboid) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
deleted file mode 100644
index 8aace22..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIRow;
-
-/**
- * @author yangli9
- *
- */
-public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable {
-
- byte[] family;
-
- HTableInterface table;
- ResultScanner scanner;
- Iterator<Result> iterator;
-
- public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family) throws IOException {
- this.family = family;
-
- this.table = hconn.getTable(tableName);
- this.scanner = table.getScanner(family);
- this.iterator = scanner.iterator();
- }
-
- @Override
- public void close() {
- IOUtils.closeQuietly(scanner);
- IOUtils.closeQuietly(table);
- }
-
- @Override
- public Iterator<IIRow> iterator() {
- return new MyIterator();
- }
-
- private class MyIterator implements Iterator<IIRow> {
-
- ImmutableBytesWritable key = new ImmutableBytesWritable();
- ImmutableBytesWritable value = new ImmutableBytesWritable();
- ImmutableBytesWritable dict = new ImmutableBytesWritable();
- IIRow pair = new IIRow(key, value, dict);
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public IIRow next() {
- Result r = iterator.next();
- Cell c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES);
- key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES);
- dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
- return pair;
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java
deleted file mode 100644
index 22c8ec6..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.ii;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-
-/**
- */
-public class IIBulkLoadJob extends AbstractHadoopJob {
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- options.addOption(OPTION_II_NAME);
- parseOptions(options, args);
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
- String input = getOptionValue(OPTION_INPUT_PATH);
-
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- FsShell shell = new FsShell(conf);
- try {
- shell.run(new String[] { "-chmod", "-R", "777", input });
- } catch (Exception e) {
- logger.error("Couldn't change the file permissions ", e);
- throw new IOException(e);
- }
-
- return ToolRunner.run(new LoadIncrementalHFiles(conf), new String[] { input, tableName });
-
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java
deleted file mode 100644
index 30dca8e..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.ii;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class IICreateHFileJob extends AbstractHadoopJob {
-
- protected static final Logger logger = LoggerFactory.getLogger(IICreateHFileJob.class);
-
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_INPUT_PATH);
- options.addOption(OPTION_OUTPUT_PATH);
- options.addOption(OPTION_HTABLE_NAME);
- parseOptions(options, args);
-
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
-
- String iiName = getOptionValue(OPTION_II_NAME);
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIInstance ii = mgr.getII(iiName);
-
- setJobClasspath(job, ii.getConfig());
-
- addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
- FileOutputFormat.setOutputPath(job, output);
-
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapperClass(IICreateHFileMapper.class);
- job.setMapOutputKeyClass(ImmutableBytesWritable.class);
- job.setMapOutputValueClass(KeyValue.class);
-
- String tableName = getOptionValue(OPTION_HTABLE_NAME);
- HTable htable = new HTable(HBaseConfiguration.create(getConf()), tableName);
- HFileOutputFormat.configureIncrementalLoad(job, htable);
-
- this.deletePath(job.getConfiguration(), output);
-
- return waitForCompletion(job);
- } catch (Exception e) {
- printUsage(options);
- throw e;
- }
- }
-
-}