You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/07/08 07:15:21 UTC
[8/8] kylin git commit: KYLIN-1858 remove all ii related code
KYLIN-1858 remove all ii related code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cc0b9c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cc0b9c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cc0b9c4
Branch: refs/heads/master
Commit: 2cc0b9c43f77624dfaa09b6410a818ac020fb475
Parents: cd5e603
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Jul 8 15:14:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Jul 8 15:14:37 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 6 -
.../kylin/job/hadoop/invertedindex/IICLI.java | 106 -
.../kylin/job/hadoop/invertedindex/IITest.java | 266 --
.../apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/common/persistence/ResourceStore.java | 2 -
.../org/apache/kylin/common/util/ClassUtil.java | 1 -
.../common/util/AbstractKylinTestCase.java | 2 -
engine-mr/pom.xml | 5 -
.../org/apache/kylin/engine/mr/IMROutput.java | 26 +-
.../kylin/engine/mr/JobBuilderSupport.java | 9 -
.../java/org/apache/kylin/engine/mr/MRUtil.java | 5 -
.../engine/mr/common/AbstractHadoopJob.java | 18 -
.../mr/invertedindex/BatchIIJobBuilder.java | 84 -
.../kylin/engine/mr/invertedindex/IIJob.java | 71 -
.../mr/invertedindex/InvertedIndexJob.java | 135 -
.../mr/invertedindex/InvertedIndexMapper.java | 77 -
.../invertedindex/InvertedIndexPartitioner.java | 73 -
.../mr/invertedindex/InvertedIndexReducer.java | 128 -
.../UpdateIIInfoAfterBuildStep.java | 83 -
engine-streaming/pom.xml | 5 -
.../invertedindex/test_kylin_ii_inner_join.json | 16 -
.../invertedindex/test_kylin_ii_left_join.json | 15 -
.../invertedindex/test_streaming_table_ii.json | 15 -
.../test_kylin_ii_inner_join_desc.json | 69 -
.../test_kylin_ii_left_join_desc.json | 69 -
.../test_streaming_table_ii_desc.json | 26 -
.../localmeta/project/default.json | 10 -
.../.settings/org.eclipse.core.resources.prefs | 6 -
.../.settings/org.eclipse.jdt.core.prefs | 386 --
.../.settings/org.eclipse.jdt.ui.prefs | 7 -
invertedindex/pom.xml | 103 -
.../invertedindex/IICapabilityChecker.java | 182 -
.../kylin/invertedindex/IIDescManager.java | 251 --
.../apache/kylin/invertedindex/IIInstance.java | 369 --
.../apache/kylin/invertedindex/IIManager.java | 272 --
.../apache/kylin/invertedindex/IISegment.java | 279 --
.../invertedindex/index/BatchSliceMaker.java | 95 -
.../invertedindex/index/BitMapContainer.java | 227 --
.../index/ColumnValueContainer.java | 43 -
.../index/CompressedValueContainer.java | 184 -
.../index/IncrementalSliceMaker.java | 123 -
.../invertedindex/index/RawTableRecord.java | 133 -
.../index/ShardingSliceBuilder.java | 53 -
.../apache/kylin/invertedindex/index/Slice.java | 215 -
.../kylin/invertedindex/index/SliceBuilder.java | 78 -
.../kylin/invertedindex/index/TableRecord.java | 193 -
.../invertedindex/index/TableRecordInfo.java | 147 -
.../index/TableRecordInfoDigest.java | 175 -
.../invertedindex/measure/FixedHLLCodec.java | 78 -
.../measure/FixedLenMeasureCodec.java | 50 -
.../measure/FixedPointLongCodec.java | 117 -
.../kylin/invertedindex/model/IIDesc.java | 418 --
.../model/IIJoinedFlatTableDesc.java | 68 -
.../invertedindex/model/IIKeyValueCodec.java | 235 --
.../model/IIKeyValueCodecWithState.java | 100 -
.../apache/kylin/invertedindex/model/IIRow.java | 90 -
.../invertedindex/model/KeyValueCodec.java | 48 -
.../invertedindex/util/IIDictionaryBuilder.java | 86 -
.../kylin/invertedindex/IIDescManagerTest.java | 103 -
.../apache/kylin/invertedindex/IIDescTest.java | 66 -
.../kylin/invertedindex/IIInstanceTest.java | 70 -
.../invertedindex/InvertedIndexLocalTest.java | 264 --
.../org/apache/kylin/invertedindex/LZFTest.java | 49 -
.../measure/FixedPointLongCodecTest.java | 62 -
kylin-it/pom.xml | 6 -
.../apache/kylin/query/ITCombinationTest.java | 1 -
.../org/apache/kylin/query/ITIIQueryTest.java | 92 -
.../apache/kylin/query/ITKylinQueryTest.java | 1 -
.../hbase/ii/ITInvertedIndexHBaseTest.java | 115 -
pom.xml | 1 -
query/pom.xml | 6 -
.../apache/kylin/query/routing/Candidate.java | 1 -
.../apache/kylin/rest/service/BasicService.java | 10 -
.../apache/kylin/rest/service/CacheService.java | 22 -
.../apache/kylin/rest/service/ModelService.java | 9 -
.../kylin/rest/service/ServiceTestBase.java | 4 -
storage-hbase/pom.xml | 6 -
.../kylin/storage/hbase/HBaseStorage.java | 17 +-
.../coprocessor/CoprocessorProjector.java | 16 +-
.../common/coprocessor/CoprocessorRowType.java | 12 -
.../hbase/cube/v1/HBaseClientKVIterator.java | 94 -
.../kylin/storage/hbase/ii/IIBulkLoadJob.java | 65 -
.../storage/hbase/ii/IICreateHFileJob.java | 88 -
.../storage/hbase/ii/IICreateHFileMapper.java | 71 -
.../storage/hbase/ii/IICreateHTableJob.java | 158 -
.../hbase/ii/InvertedIndexStorageQuery.java | 67 -
.../endpoint/BitMapFilterEvaluator.java | 249 --
.../endpoint/ClearTextDictionary.java | 66 -
.../endpoint/EndpointAggregationCache.java | 46 -
.../endpoint/EndpointAggregators.java | 287 --
.../coprocessor/endpoint/EndpointEnabler.java | 53 -
.../endpoint/EndpointTupleConverter.java | 94 -
.../endpoint/EndpointTupleIterator.java | 364 --
.../endpoint/HbaseServerKVIterator.java | 104 -
.../ii/coprocessor/endpoint/IIEndpoint.java | 336 --
.../coprocessor/endpoint/IIResponseAdapter.java | 24 -
.../coprocessor/endpoint/LocalDictionary.java | 74 -
.../endpoint/SliceBitMapProvider.java | 55 -
.../endpoint/generated/IIProtos.java | 3773 ------------------
.../ii/coprocessor/endpoint/protobuf/II.proto | 65 -
.../storage/hbase/steps/HBaseMROutput.java | 18 -
.../kylin/storage/hbase/steps/HBaseMRSteps.java | 78 -
.../hbase/util/DeployCoprocessorCLI.java | 35 +-
.../hbase/util/IIDeployCoprocessorCLI.java | 157 -
.../storage/hbase/util/StorageCleanupJob.java | 16 -
.../storage/hbase/util/UpdateHTableHostCLI.java | 15 -
.../hbase/common/TsConditionEraserTest.java | 160 -
.../endpoint/BitMapFilterEvaluatorTest.java | 228 --
.../endpoint/EndpointAggregationTest.java | 177 -
.../endpoint/TableRecordInfoTest.java | 63 -
.../endpoint/TsConditionExtractorTest.java | 214 -
111 files changed, 11 insertions(+), 14351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 3904225..fc1ef63 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -56,12 +56,6 @@
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java
deleted file mode 100644
index 7e7be34..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.IIRow;
-
-/**
- * @author yangli9
- */
-public class IICLI {
-
- public static void main(String[] args) throws IOException {
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- String iiName = args[0];
- IIInstance ii = mgr.getII(iiName);
-
- String path = args[1];
- System.out.println("Reading from " + path + " ...");
-
- TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment());
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- int count = 0;
- for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) {
- for (RawTableRecord rec : slice) {
- System.out.printf(new TableRecord(rec, info).toString());
- count++;
- }
- }
- System.out.println("Total " + count + " records");
- }
-
- public static Iterable<IIRow> readSequenceKVs(Configuration hconf, String path) throws IOException {
- final Reader reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path)));
- return new Iterable<IIRow>() {
- @Override
- public Iterator<IIRow> iterator() {
- return new Iterator<IIRow>() {
- ImmutableBytesWritable k = new ImmutableBytesWritable();
- ImmutableBytesWritable v = new ImmutableBytesWritable();
- IIRow pair = new IIRow(k, v, null);
-
- @Override
- public boolean hasNext() {
- boolean hasNext = false;
- try {
- hasNext = reader.next(k, v);
- } catch (IOException e) {
- throw new RuntimeException(e);
- } finally {
- if (hasNext == false) {
- IOUtils.closeQuietly(reader);
- }
- }
- return hasNext;
- }
-
- @Override
- public IIRow next() {
- return pair;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
deleted file mode 100644
index 3390586..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.invertedindex;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Queue;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.FIFOIterable;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.SliceBuilder;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodecWithState;
-import org.apache.kylin.invertedindex.model.IIRow;
-import org.apache.kylin.invertedindex.model.KeyValueCodec;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.kafka.StreamingParser;
-import org.apache.kylin.source.kafka.StringStreamingParser;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
-import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.ClearTextDictionary;
-import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.EndpointAggregators;
-import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.IIEndpoint;
-import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import kafka.message.Message;
-import kafka.message.MessageAndOffset;
-
-/**
- */
-public class IITest extends LocalFileMetadataTestCase {
-
- String iiName = "test_kylin_ii_inner_join";
- IIInstance ii;
- IIDesc iiDesc;
-
- List<IIRow> iiRows;
-
- final String[] inputData = new String[] { //
- "FP-non GTC,0,15,145970,0,28,Toys,2008-10-08 07:18:40,USER_Y,Toys & Hobbies,Models & Kits,Automotive,0,Ebay,USER_S,15,Professional-Other,2012-08-16,2012-08-11,0,2012-08-16,145970,10000329,26.8551,0", //
- "ABIN,0,-99,43479,0,21,Photo,2012-09-11 20:26:04,USER_Y,Cameras & Photo,Film Photography,Other,0,Ebay,USER_S,-99,Not Applicable,2012-08-16,2012-08-11,0,2012-08-16,43479,10000807,26.2474,0", //
- "ABIN,0,16,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,16,Consumer-Other,2012-08-16,2012-08-11,0,2012-08-16,80053,10000261,94.2273,0" };
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- this.ii = IIManager.getInstance(getTestConfig()).getII(iiName);
- this.iiDesc = ii.getDescriptor();
-
- List<MessageAndOffset> messages = Lists.transform(Arrays.asList(inputData), new Function<String, MessageAndOffset>() {
- @Nullable
- @Override
- public MessageAndOffset apply(String input) {
- return new MessageAndOffset(new Message(input.getBytes()), System.currentTimeMillis());
- }
- });
-
- final StreamingParser parser = StringStreamingParser.instance;
- final List<StreamingMessage> streamingMessages = Lists.transform(messages, new Function<MessageAndOffset, StreamingMessage>() {
- @Nullable
- @Override
- public StreamingMessage apply(@Nullable MessageAndOffset input) {
- return parser.parse(input);
- }
- });
- StreamingBatch batch = new StreamingBatch(streamingMessages, Pair.newPair(0L, System.currentTimeMillis()));
-
- iiRows = Lists.newArrayList();
- final Slice slice = new SliceBuilder(iiDesc, (short) 0).buildSlice((batch));
- IIKeyValueCodec codec = new IIKeyValueCodec(slice.getInfo());
- for (IIRow iiRow : codec.encodeKeyValue(slice)) {
- iiRows.add(iiRow);
- }
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- /**
- * simulate stream building into slices, and encode the slice into IIRows.
- * Then reconstruct the IIRows to slice.
- */
- @Test
- public void basicTest() {
- Queue<IIRow> buffer = Lists.newLinkedList();
- FIFOIterable bufferIterable = new FIFOIterable(buffer);
- TableRecordInfo info = new TableRecordInfo(iiDesc);
- TableRecordInfoDigest digest = info.getDigest();
- KeyValueCodec codec = new IIKeyValueCodecWithState(digest);
- Iterator<Slice> slices = codec.decodeKeyValue(bufferIterable).iterator();
-
- Assert.assertTrue(!slices.hasNext());
- Assert.assertEquals(iiRows.size(), digest.getColumnCount());
-
- for (int i = 0; i < digest.getColumnCount(); ++i) {
- buffer.add(iiRows.get(i));
-
- if (i != digest.getColumnCount() - 1) {
- Assert.assertTrue(!slices.hasNext());
- } else {
- Assert.assertTrue(slices.hasNext());
- }
- }
-
- Slice newSlice = slices.next();
- Assert.assertEquals(newSlice.getLocalDictionaries()[0].getSize(), 2);
- }
-
- @Test
- public void IIEndpointTest() {
- TableRecordInfo info = new TableRecordInfo(ii.getDescriptor());
- if (ii.getFirstSegment() == null) {
- IISegment segment = IIManager.getInstance(getTestConfig()).buildSegment(ii, 0, System.currentTimeMillis());
- ii.getSegments().add(segment);
- }
- CoprocessorRowType type = CoprocessorRowType.fromTableRecordInfo(info, ii.getFirstSegment().getColumns());
- CoprocessorProjector projector = CoprocessorProjector.makeForEndpoint(info, Collections.singletonList(ii.getDescriptor().findColumnRef("default.test_kylin_fact", "lstg_format_name")));
-
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("SUM");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue("PRICE");
- f1.setParameter(p1);
- f1.setReturnType("decimal(19,4)");
-
- TblColRef column = ii.getDescriptor().findColumnRef("default.test_kylin_fact", "cal_dt");
- CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE);
- ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter);
- ConstantTupleFilter constantFilter = null;
- constantFilter = new ConstantTupleFilter(("2012-08-16"));
- compareFilter.addChild(constantFilter);
-
- EndpointAggregators aggregators = EndpointAggregators.fromFunctions(info, Collections.singletonList(f1));
- CoprocessorFilter filter = CoprocessorFilter.fromFilter(new ClearTextDictionary(info), compareFilter, FilterDecorator.FilterConstantsTreatment.AS_IT_IS);
-
- final Iterator<IIRow> iiRowIterator = iiRows.iterator();
-
- IIEndpoint endpoint = new IIEndpoint();
- IIProtos.IIResponseInternal response = endpoint.getResponse(new RegionScanner() {
- @Override
- public HRegionInfo getRegionInfo() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean isFilterDone() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean reseek(byte[] row) throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public long getMaxResultSize() {
- throw new NotImplementedException();
-
- }
-
- @Override
- public long getMvccReadPoint() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean nextRaw(List<Cell> result) throws IOException {
- if (iiRowIterator.hasNext()) {
- IIRow iiRow = iiRowIterator.next();
- result.addAll(iiRow.makeCells());
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean next(List<Cell> results) throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public void close() throws IOException {
- throw new NotImplementedException();
- }
- }, type, projector, aggregators, filter);
-
- Assert.assertEquals(2, response.getRowsList().size());
- System.out.println(response.getRowsList().size());
- Set<String> answers = Sets.newHashSet("120.4747", "26.8551");
- for (IIProtos.IIResponseInternal.IIRow responseRow : response.getRowsList()) {
- ByteBuffer bf = responseRow.getMeasures().asReadOnlyByteBuffer();
- List<Object> metrics = aggregators.deserializeMetricValues(bf);
- Assert.assertTrue(answers.contains(metrics.get(0)));
- answers.remove(metrics.get(0));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 84c4db9..bfbaaa4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -203,7 +203,7 @@ abstract public class KylinConfigBase implements Serializable {
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.realization.providers", //
- new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.invertedindex.IIManager" });
+ new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
}
public CliCommandExecutor getCliCommandExecutor() throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index cf00c05..99a9695 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -42,9 +42,7 @@ abstract public class ResourceStore {
private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class);
public static final String CUBE_RESOURCE_ROOT = "/cube";
- public static final String II_RESOURCE_ROOT = "/invertedindex";
public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc";
- public static final String II_DESC_RESOURCE_ROOT = "/invertedindex_desc";
public static final String DATA_MODEL_DESC_RESOURCE_ROOT = "/model_desc";
public static final String DICT_RESOURCE_ROOT = "/dict";
public static final String PROJECT_RESOURCE_ROOT = "/project";
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index ef89754..3c63c41 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -54,7 +54,6 @@ public class ClassUtil {
classRenameMap.put("org.apache.kylin.job.common.HadoopShellExecutable", "org.apache.kylin.engine.mr.common.HadoopShellExecutable");
classRenameMap.put("org.apache.kylin.job.common.MapReduceExecutable", "org.apache.kylin.engine.mr.common.MapReduceExecutable");
classRenameMap.put("org.apache.kylin.job.cube.CubingJob", "org.apache.kylin.engine.mr.CubingJob");
- classRenameMap.put("org.apache.kylin.job.invertedindex.IIJob", "org.apache.kylin.engine.mr.invertedindex.IIJob");
classRenameMap.put("org.apache.kylin.job.cube.GarbageCollectionStep", "org.apache.kylin.storage.hbase.steps.DeprecatedGCStep");
classRenameMap.put("org.apache.kylin.job.cube.MergeDictionaryStep", "org.apache.kylin.engine.mr.steps.MergeDictionaryStep");
classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep");
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 684d828..556019f 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -31,8 +31,6 @@ public abstract class AbstractKylinTestCase {
public static final String[] SERVICES_WITH_CACHE = { //
"org.apache.kylin.cube.CubeManager", //
"org.apache.kylin.cube.CubeDescManager", //
- "org.apache.kylin.invertedindex.IIDescManager", //
- "org.apache.kylin.invertedindex.IIManager", //
"org.apache.kylin.storage.hybrid.HybridManager", //
"org.apache.kylin.metadata.realization.RealizationRegistry", //
"org.apache.kylin.metadata.project.ProjectManager", //
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 363e459..12a6b73 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -50,11 +50,6 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-job</artifactId>
<version>${project.parent.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
index 276af65..beebb4e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java
@@ -19,7 +19,6 @@
package org.apache.kylin.engine.mr;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
public interface IMROutput {
@@ -27,11 +26,8 @@ public interface IMROutput {
/** Return a helper to participate in batch cubing job flow. */
public IMRBatchCubingOutputSide getBatchCubingOutputSide(CubeSegment seg);
- /** Return a helper to participate in batch cubing job flow. */
- public IMRBatchInvertedIndexingOutputSide getBatchInvertedIndexingOutputSide(IISegment seg);
-
/**
- * Participate the batch cubing flow as the output side. Responsible for saving
+ * Participate the batch cubing flow as the output side. Responsible for savg
* the cuboid output to storage (Phase 3).
*
* - Phase 1: Create Flat Table
@@ -80,24 +76,4 @@ public interface IMROutput {
public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow);
}
- /**
- * Participate the batch inverted indexing flow as the output side. Responsible for saving
- * the output to storage (Phase 3).
- *
- * - Phase 1: Create Flat Table
- * - Phase 2: Build Dictionary
- * - Phase 3: Build II
- * - Phase 4: Update Metadata & Cleanup
- */
- public interface IMRBatchInvertedIndexingOutputSide {
-
- /**
- * Add step that saves II output from HDFS to storage.
- *
- */
- public void addStepPhase3_BuildII(DefaultChainedExecutable jobFlow, String rootPath);
-
- /** Add step that does any necessary clean up. */
- public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow);
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 5228088..5a82fe5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -25,7 +25,6 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.engine.mr.invertedindex.UpdateIIInfoAfterBuildStep;
import org.apache.kylin.engine.mr.steps.CreateDictionaryJob;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob;
@@ -130,14 +129,6 @@ public class JobBuilderSupport {
return result;
}
- public UpdateIIInfoAfterBuildStep createUpdateIIInfoAfterBuildStep(String jobId) {
- final UpdateIIInfoAfterBuildStep updateIIInfoStep = new UpdateIIInfoAfterBuildStep();
- updateIIInfoStep.setName(ExecutableConstants.STEP_NAME_UPDATE_II_INFO);
- updateIIInfoStep.setInvertedIndexName(seg.getRealization().getName());
- updateIIInfoStep.setJobId(jobId);
- return updateIIInfoStep;
- }
-
// ============================================================================
public String getJobWorkingDir(String jobId) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index b67dc38..abe05d7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -29,7 +29,6 @@ import org.apache.kylin.engine.mr.IMROutput.IMRBatchCubingOutputSide;
import org.apache.kylin.engine.mr.IMROutput.IMRBatchMergeOutputSide;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchMergeOutputSide2;
-import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.realization.IRealizationSegment;
@@ -70,10 +69,6 @@ public class MRUtil {
return StorageFactory.createEngineAdapter(seg, IMROutput2.class).getBatchMergeOutputSide(seg);
}
- public static IMROutput.IMRBatchInvertedIndexingOutputSide getBatchInvertedIndexingOutputSide(IISegment seg) {
- return StorageFactory.createEngineAdapter(seg, IMROutput.class).getBatchInvertedIndexingOutputSide(seg);
- }
-
// use this method instead of ToolRunner.run() because ToolRunner.run() is not thread-sale
// Refer to: http://stackoverflow.com/questions/22462665/is-hadoops-toorunner-thread-safe
public static int runMRJob(Tool tool, String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 9b5ed67..bafe65b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -63,7 +63,6 @@ import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.MetadataManager;
@@ -441,23 +440,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf);
}
- protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
- MetadataManager metaMgr = MetadataManager.getInstance(ii.getConfig());
-
- // write II / model_desc / II_desc / dict / table
- ArrayList<String> dumpList = new ArrayList<String>();
- dumpList.add(ii.getResourcePath());
- dumpList.add(ii.getDescriptor().getModel().getResourcePath());
- dumpList.add(ii.getDescriptor().getResourcePath());
-
- for (String tableName : ii.getDescriptor().getModel().getAllTables()) {
- TableDesc table = metaMgr.getTableDesc(tableName);
- dumpList.add(table.getResourcePath());
- }
-
- attachKylinPropsAndMetadata(dumpList, ii.getConfig(), conf);
- }
-
protected void attachKylinPropsAndMetadata(ArrayList<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
deleted file mode 100644
index e582fa0..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide;
-import org.apache.kylin.engine.mr.IMROutput;
-import org.apache.kylin.engine.mr.JobBuilderSupport;
-import org.apache.kylin.engine.mr.MRUtil;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class BatchIIJobBuilder extends JobBuilderSupport {
-
- private static final Logger logger = LoggerFactory.getLogger(BatchIIJobBuilder.class);
-
- private final IMRBatchCubingInputSide inputSide;
- private final IMROutput.IMRBatchInvertedIndexingOutputSide outputSide;
-
- public BatchIIJobBuilder(IISegment newSegment, String submitter) {
- super(newSegment, submitter);
- this.inputSide = MRUtil.getBatchCubingInputSide(newSegment);
- this.outputSide = MRUtil.getBatchInvertedIndexingOutputSide(newSegment);
- }
-
- public IIJob build() {
- logger.info("MR new job to BUILD segment " + seg);
-
- final IIJob result = IIJob.createBuildJob((IISegment) seg, submitter, config);
- final String jobId = result.getId();
-
- final String iiRootPath = getRealizationRootPath(jobId) + "/";
- // Phase 1: Create Flat Table
- inputSide.addStepPhase1_CreateFlatTable(result);
-
- // Phase 2: Build Inverted Index
- result.addTask(createInvertedIndexStep((IISegment) seg, iiRootPath));
- outputSide.addStepPhase3_BuildII(result, iiRootPath);
-
- // Phase 3: Update Metadata & Cleanup
- result.addTask(createUpdateIIInfoAfterBuildStep(jobId));
- inputSide.addStepPhase4_Cleanup(result);
- outputSide.addStepPhase4_Cleanup(result);
-
- return result;
- }
-
- private MapReduceExecutable createInvertedIndexStep(IISegment seg, String iiOutputTempPath) {
- MapReduceExecutable buildIIStep = new MapReduceExecutable();
-
- StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd);
-
- buildIIStep.setName(ExecutableConstants.STEP_NAME_BUILD_II);
-
- appendExecCmdParameters(cmd, BatchConstants.ARG_II_NAME, seg.getRealization().getName());
- appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, iiOutputTempPath);
- appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, ExecutableConstants.STEP_NAME_BUILD_II);
-
- buildIIStep.setMapReduceParams(cmd.toString());
- buildIIStep.setMapReduceJobClass(InvertedIndexJob.class);
- return buildIIStep;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java
deleted file mode 100644
index 7c0748a..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-
-/**
- */
-public class IIJob extends DefaultChainedExecutable {
-
- public IIJob() {
- super();
- }
-
- private static final String II_INSTANCE_NAME = "iiName";
- private static final String SEGMENT_ID = "segmentId";
-
- void setIIName(String name) {
- setParam(II_INSTANCE_NAME, name);
- }
-
- public String getIIName() {
- return getParam(II_INSTANCE_NAME);
- }
-
- void setSegmentId(String segmentId) {
- setParam(SEGMENT_ID, segmentId);
- }
-
- public String getSegmentId() {
- return getParam(SEGMENT_ID);
- }
-
- public static IIJob createBuildJob(IISegment seg, String submitter, JobEngineConfig config) {
- return initialJob(seg, "BUILD", submitter, config);
- }
-
- private static IIJob initialJob(IISegment seg, String type, String submitter, JobEngineConfig config) {
- IIJob result = new IIJob();
- SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
- result.setIIName(seg.getIIInstance().getName());
- result.setSegmentId(seg.getUuid());
- result.setName(seg.getIIInstance().getName() + " - " + seg.getName() + " - " + type + " - " + format.format(new Date(System.currentTimeMillis())));
- result.setSubmitter(submitter);
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
deleted file mode 100644
index f7adf6b..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.IMRInput;
-import org.apache.kylin.engine.mr.MRUtil;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class InvertedIndexJob extends AbstractHadoopJob {
- protected static final Logger logger = LoggerFactory.getLogger(InvertedIndexJob.class);
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
-
- try {
- options.addOption(OPTION_JOB_NAME);
- options.addOption(OPTION_II_NAME);
- options.addOption(OPTION_OUTPUT_PATH);
- parseOptions(options, args);
-
- job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- String iiname = getOptionValue(OPTION_II_NAME);
- Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
-
- // ----------------------------------------------------------------------------
-
- logger.info("Starting: " + job.getJobName());
-
- IIInstance ii = getII(iiname);
- short sharding = ii.getDescriptor().getSharding();
-
- setJobClasspath(job, ii.getConfig());
-
- setupMapper(ii.getFirstSegment());
- setupReducer(output, sharding);
- attachMetadata(ii);
-
- return waitForCompletion(job);
-
- } catch (Exception e) {
- printUsage(options);
- throw e;
- } finally {
- if (job != null)
- cleanupTempConfFile(job.getConfiguration());
- }
-
- }
-
- private IIInstance getII(String iiName) {
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIInstance ii = mgr.getII(iiName);
- if (ii == null)
- throw new IllegalArgumentException("No Inverted Index found by name " + iiName);
- return ii;
- }
-
- private void attachMetadata(IIInstance ii) throws IOException {
-
- Configuration conf = job.getConfiguration();
- attachKylinPropsAndMetadata(ii, conf);
-
- IISegment seg = ii.getFirstSegment();
- conf.set(BatchConstants.CFG_II_NAME, ii.getName());
- conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName());
- }
-
- private void setupMapper(IISegment segment) throws IOException {
-
- IMRInput.IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(segment).getFlatTableInputFormat();
- flatTableInputFormat.configureJob(job);
-
- job.setMapperClass(InvertedIndexMapper.class);
- job.setMapOutputKeyClass(LongWritable.class);
- job.setPartitionerClass(InvertedIndexPartitioner.class);
- }
-
- private void setupReducer(Path output, short sharding) throws IOException {
- job.setReducerClass(InvertedIndexReducer.class);
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(ImmutableBytesWritable.class);
- job.setOutputValueClass(ImmutableBytesWritable.class);
-
- job.setNumReduceTasks(sharding);
-
- FileOutputFormat.setOutputPath(job, output);
-
- job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
-
- deletePath(job.getConfiguration(), output);
- }
-
- public static void main(String[] args) throws Exception {
- InvertedIndexJob job = new InvertedIndexJob();
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java
deleted file mode 100644
index 27e2470..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.engine.mr.IMRInput;
-import org.apache.kylin.engine.mr.KylinMapper;
-import org.apache.kylin.engine.mr.MRUtil;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexMapper<KEYIN> extends KylinMapper<KEYIN, Object, LongWritable, Writable> {
-
- private TableRecordInfo info;
-
- private LongWritable outputKey;
- private IMRInput.IMRTableInputFormat flatTableInputFormat;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.bindCurrentConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
- IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
- this.info = new TableRecordInfo(seg);
-
- outputKey = new LongWritable();
-
- flatTableInputFormat = MRUtil.getBatchCubingInputSide(ii.getFirstSegment()).getFlatTableInputFormat();
- }
-
- @Override
- public void map(KEYIN key, Object record, Context context) throws IOException, InterruptedException {
-
- Writable writableRecord = (Writable) record;
- String[] row = flatTableInputFormat.parseMapperInput(writableRecord);
- String timestampString = row[info.getTimestampColumn()];
-
- outputKey.set(DateFormat.stringToMillis(timestampString));
- //
- context.write(outputKey, writableRecord);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java
deleted file mode 100644
index dcf707f..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexPartitioner extends Partitioner<LongWritable, ImmutableBytesWritable> implements Configurable {
-
- private Configuration conf;
- private TableRecordInfo info;
- private TableRecord rec;
-
- @Override
- public int getPartition(LongWritable key, ImmutableBytesWritable value, int numPartitions) {
- rec.setBytes(value.get(), value.getOffset(), value.getLength());
- return rec.getShard();
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- try {
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
- IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
- this.info = new TableRecordInfo(seg);
- this.rec = this.info.createTableRecord();
- } catch (IOException e) {
- throw new RuntimeException("", e);
- }
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
deleted file mode 100644
index e7e760a..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.common.util.StreamingMessage;
-import org.apache.kylin.engine.mr.IMRInput;
-import org.apache.kylin.engine.mr.KylinReducer;
-import org.apache.kylin.engine.mr.MRUtil;
-import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.SliceBuilder;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.IIRow;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class InvertedIndexReducer extends KylinReducer<LongWritable, Object, ImmutableBytesWritable, ImmutableBytesWritable> {
-
- private TableRecordInfo info;
- private IIKeyValueCodec kv;
- private IMRInput.IMRTableInputFormat flatTableInputFormat;
- private SliceBuilder sliceBuilder;
- private ArrayList<StreamingMessage> messages;
- private int sliceSize;
- private ImmutableBytesWritable immutableBytesWritable;
- private ByteBuffer valueBuf;
-
- @Override
- protected void setup(Context context) throws IOException {
- super.bindCurrentConfiguration(context.getConfiguration());
-
- Configuration conf = context.getConfiguration();
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
- IIManager mgr = IIManager.getInstance(config);
- IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
- IISegment seg = ii.getFirstSegment();
- info = new TableRecordInfo(seg);
- kv = new IIKeyValueCodec(info.getDigest());
- flatTableInputFormat = MRUtil.getBatchCubingInputSide(ii.getFirstSegment()).getFlatTableInputFormat();
- sliceSize = ii.getDescriptor().getSliceSize();
- short shard = (short) context.getTaskAttemptID().getTaskID().getId();
- System.out.println("Generating to shard - " + shard);
- sliceBuilder = new SliceBuilder(seg.getIIDesc(), shard);
- messages = Lists.newArrayListWithCapacity(sliceSize);
- immutableBytesWritable = new ImmutableBytesWritable();
- valueBuf = ByteBuffer.allocate(1024 * 1024); // 1MB
- }
-
- @Override
- public void reduce(LongWritable key, Iterable<Object> values, Context context) //
- throws IOException, InterruptedException {
- for (Object v : values) {
- String[] row = flatTableInputFormat.parseMapperInput(v);
- messages.add((parse(row)));
- if (messages.size() >= sliceSize) {
- buildAndOutput(new StreamingBatch(messages, Pair.newPair(System.currentTimeMillis(), System.currentTimeMillis())), context);
- messages = Lists.newArrayList();
- }
- }
- }
-
- private StreamingMessage parse(String[] row) {
- return new StreamingMessage(Lists.newArrayList(row), System.currentTimeMillis(), System.currentTimeMillis(), Collections.<String, Object> emptyMap());
- }
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- if (!messages.isEmpty()) {
- buildAndOutput(new StreamingBatch(messages, Pair.newPair(System.currentTimeMillis(), System.currentTimeMillis())), context);
- messages.clear();
- }
-
- }
-
- private void buildAndOutput(StreamingBatch streamingBatch, Context context) throws IOException, InterruptedException {
- final Slice slice = sliceBuilder.buildSlice(streamingBatch);
- ImmutableBytesWritable value, dictionary;
- for (IIRow pair : kv.encodeKeyValue(slice)) {
- value = pair.getValue();
- dictionary = pair.getDictionary();
- int newLength = 4 + value.getLength() + dictionary.getLength();
- if (newLength > valueBuf.limit()) {
- valueBuf = ByteBuffer.allocate(newLength);
- }
- valueBuf.clear();
- valueBuf.putInt(value.getLength());
- valueBuf.put(value.get(), value.getOffset(), value.getLength());
- valueBuf.put(dictionary.get(), dictionary.getOffset(), dictionary.getLength());
- immutableBytesWritable.set(valueBuf.array(), 0, newLength);
- context.write(pair.getKey(), immutableBytesWritable);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java
deleted file mode 100644
index a509c1f..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class UpdateIIInfoAfterBuildStep extends AbstractExecutable {
- private static final Logger logger = LoggerFactory.getLogger(UpdateIIInfoAfterBuildStep.class);
-
- private static final String II_NAME = "iiName";
- private static final String JOB_ID = "jobId";
-
- public UpdateIIInfoAfterBuildStep() {
- super();
- }
-
- public void setInvertedIndexName(String cubeName) {
- this.setParam(II_NAME, cubeName);
- }
-
- private String getInvertedIndexName() {
- return getParam(II_NAME);
- }
-
- public void setJobId(String id) {
- setParam(JOB_ID, id);
- }
-
- private String getJobId() {
- return getParam(JOB_ID);
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
-
- IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv());
- IIInstance ii = mgr.getII(getInvertedIndexName());
- IISegment segment = ii.getFirstSegment();
- segment.setStatus(SegmentStatusEnum.READY);
-
- segment.setLastBuildJobID(getJobId());
- segment.setLastBuildTime(System.currentTimeMillis());
-
- try {
- mgr.updateII(ii);
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- } catch (IOException e) {
- logger.error("fail to update inverted index after build", e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
index 54955a6..7110377 100644
--- a/engine-streaming/pom.xml
+++ b/engine-streaming/pom.xml
@@ -43,11 +43,6 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
<version>${project.parent.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
deleted file mode 100644
index 13fc282..0000000
--- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-
- "uuid": "dk853e80-41be-49a5-89ca-9fb7294db186",
- "name": "test_kylin_ii_inner_join",
- "owner": null,
- "version": null,
- "cost": 10,
- "status": "DISABLED",
- "segments": [],
- "last_modified": 1420016227424,
- "descriptor": "test_kylin_ii_inner_join_desc",
- "create_time": null,
- "input_records_size": 0,
- "size_kb": 0,
- "source_records_count": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
deleted file mode 100644
index 28b23ef..0000000
--- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-
- "uuid": "daa53e80-41be-49a5-89ca-9fb7294db186",
- "name": "test_kylin_ii_left_join",
- "owner": null,
- "cost": 10,
- "status": "DISABLED",
- "segments": [],
- "last_modified": 1420016227424,
- "descriptor": "test_kylin_ii_left_join_desc",
- "create_time": null,
- "input_records_size": 0,
- "size_kb": 0,
- "source_records_count": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
deleted file mode 100644
index 4ec2711..0000000
--- a/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-
- "uuid": "daa53e80-41be-49a5-89ca-9fb729999812",
- "name": "test_streaming_table_ii",
- "owner": null,
- "cost": 10,
- "status": "DISABLED",
- "segments": [],
- "last_modified": 0,
- "descriptor": "test_streaming_table_ii_desc",
- "create_time": null,
- "input_records_size": 0,
- "size_kb": 0,
- "source_records_count": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
deleted file mode 100644
index eb383fb..0000000
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-
- "uuid": "88bf87b5-c7b5-4420-a12a-07f6b37b3187",
- "last_modified": 0,
- "name": "test_kylin_ii_inner_join_desc",
- "model_name": "test_kylin_inner_join_model_desc",
- "timestamp_dimension": "cal_dt",
- "value_dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- },
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT"
- ],
- "sharding": 1,
- "slice_size": 1000
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
deleted file mode 100644
index 2f152ad..0000000
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-
- "uuid": "74bf87b5-c7b5-4420-a12a-07f6b37b3187",
- "last_modified": 0,
- "name": "test_kylin_ii_left_join_desc",
- "model_name": "test_kylin_left_join_model_desc",
- "timestamp_dimension": "cal_dt",
- "value_dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- },
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT"
- ],
- "sharding": 1,
- "slice_size": 1000
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
deleted file mode 100644
index c39ae90..0000000
--- a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-
- "uuid": "74bf87b5-c7b5-4420-a12a-07f6b3864789",
- "last_modified": 0,
- "name": "test_streaming_table_ii_desc",
- "model_name": "test_streaming_table_model_desc",
- "timestamp_dimension": "minute_start",
- "value_dimensions": [
- {
- "table": "default.streaming_table",
- "columns": [
- "minute_start",
- "hour_start",
- "day_start",
- "itm",
- "site"
- ]
- }
- ],
- "metrics": [
- "gmv",
- "item_count"
- ],
- "sharding": 1,
- "slice_size": 1000
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 88538c9..3026ff8 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -24,16 +24,6 @@
"realization": "test_kylin_cube_without_slr_left_join_empty"
},
{
- "name": "test_kylin_ii_left_join",
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii_left_join"
- },
- {
- "name": "test_kylin_ii_inner_join",
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii_inner_join"
- },
- {
"name": "test_streaming_table_cube",
"type": "CUBE",
"realization": "test_streaming_table_cube"
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/invertedindex/.settings/org.eclipse.core.resources.prefs b/invertedindex/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 29abf99..0000000
--- a/invertedindex/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8