You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/14 15:15:51 UTC
[16/51] [partial] incubator-kylin git commit: cleanup for migration
from github.com
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java
deleted file mode 100644
index ed698c4..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.AbstractKylinTestCase;
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob() throws Exception {
- String input = "src/test/resources/data/flat_table/";
- String output = "target/test-output/base_cuboid/";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "base_cuboid_job";
- String level = "0";
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new BaseCuboidJob(), args));
- }
-
- @Test
- public void testJobWithBadParas() throws Exception {
-
- final String input = "src/test/resources/data/flat_table/";
- final String output = "target/test-output/base_cuboid/";
- final String metadata = AbstractKylinTestCase.LOCALMETA_TEST_DATA;
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-output", output, "-metadata", metadata };
- assertEquals(2, ToolRunner.run(conf, new BaseCuboidJob(), args));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
deleted file mode 100644
index bab08ec..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class BaseCuboidMapperPerformanceTest {
-
- String metadataUrl = "hbase:yadesk00:2181:/hbase-unsecure";
- String cubeName = "test_kylin_cube_with_slr";
- Path srcPath = new Path("/download/test_kylin_cube_with_slr_intermediate_table_64mb.seq");
-
- @Ignore
- @Test
- public void test() throws IOException, InterruptedException {
- Configuration hconf = new Configuration();
- BaseCuboidMapper mapper = new BaseCuboidMapper();
- Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
-
- mapper.setup(context);
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- while (reader.next(key, value)) {
- mapper.map(key, value, context);
- }
-
- reader.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java
deleted file mode 100644
index 95f255f..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.math.BigDecimal;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeInstance;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.kv.RowKeyDecoder;
-import com.kylinolap.cube.measure.MeasureCodec;
-import com.kylinolap.job.constant.BatchConstants;
-import com.kylinolap.metadata.model.cube.MeasureDesc;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
-
- MapDriver<Text, Text, Text, Text> mapDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapperWithHeader() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.331"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33", 1);
- }
-
- private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3, long m4) {
- MeasureCodec codec = new MeasureCodec(measures);
- Object[] values = new Object[measures.size()];
- codec.decode(valueBytes, values);
- assertTrue(new BigDecimal(m1).equals(values[0]));
- assertTrue(new BigDecimal(m2).equals(values[1]));
- assertTrue(new BigDecimal(m3).equals(values[2]));
- assertTrue(m4 == ((LongWritable) values[3]).get());
- }
-
- @Test
- public void testMapperWithNull() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N\\N"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", 1L);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java
deleted file mode 100644
index 7645222..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * @author yangli9
- *
- */
-public class CopySeq {
-
- public static void main(String[] args) throws IOException {
- copyTo64MB(args[0], args[1]);
- }
-
- public static void copyTo64MB(String src, String dst) throws IOException {
- Configuration hconf = new Configuration();
- Path srcPath = new Path(src);
- Path dstPath = new Path(dst);
-
- FileSystem fs = FileSystem.get(hconf);
- long srcSize = fs.getFileStatus(srcPath).getLen();
- int copyTimes = (int) (67108864 / srcSize); // 64 MB
- System.out.println("Copy " + copyTimes + " times");
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- Writer writer = SequenceFile.createWriter(hconf, Writer.file(dstPath), Writer.keyClass(key.getClass()), Writer.valueClass(Text.class), Writer.compression(CompressionType.BLOCK, getLZOCodec(hconf)));
-
- int count = 0;
- while (reader.next(key, value)) {
- for (int i = 0; i < copyTimes; i++) {
- writer.append(key, value);
- count++;
- }
- }
-
- System.out.println("Len: " + writer.getLength());
- System.out.println("Rows: " + count);
-
- reader.close();
- writer.close();
- }
-
- static CompressionCodec getLZOCodec(Configuration hconf) {
- CompressionCodecFactory factory = new CompressionCodecFactory(hconf);
- return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java
deleted file mode 100644
index ffe2fad..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.kv.RowConstants;
-import com.kylinolap.cube.measure.MeasureCodec;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
-
- String cubeName = "test_kylin_cube_with_slr_ready";
-
- MeasureCodec codec;
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- Object[] outKV = new Object[2];
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
- CubeDesc desc = CubeManager.getInstance(this.getTestConfig()).getCube(cubeName).getDescriptor();
- codec = new MeasureCodec(desc.getMeasures());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testBasic() throws Exception {
-
- Configuration hconf = new Configuration();
- Context context = MockupMapContext.create(hconf, this.getTestConfig().getMetadataUrl(), cubeName, outKV);
-
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapper.setup(context);
-
- Text key = new Text("not important");
- Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
-
- mapper.map(key, value, context);
-
- ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
- KeyValue outValue = (KeyValue) outKV[1];
-
- assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(), outKey.getLength()) == 0);
-
- assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(), outValue.getValueOffset(), outValue.getValueLength()) == 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java
deleted file mode 100644
index 0f3aff9..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.kylinolap.job.constant.BatchConstants;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CubeHFileMapperTest {
-
- MapDriver<Text, Text, ImmutableBytesWritable, KeyValue> mapDriver;
-
- private String cube_name = "FLAT_ITEM_CUBE";
-
- @Before
- public void setUp() {
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- @Ignore
- public void testMapper2() throws IOException {
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name);
-
- mapDriver.addInput(new Text("52010tech"), new Text("35.432"));
-
- List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run();
-
- assertEquals(2, result.size());
-
- byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 };
- ImmutableBytesWritable key = new ImmutableBytesWritable(bytes);
-
- Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0);
- Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1);
-
- assertEquals(key, p1.getFirst());
- assertEquals("cf1", new String(p1.getSecond().getFamily()));
- assertEquals("usd_amt", new String(p1.getSecond().getQualifier()));
- assertEquals("35.43", new String(p1.getSecond().getValue()));
-
- assertEquals(key, p2.getFirst());
- assertEquals("cf1", new String(p2.getSecond().getFamily()));
- assertEquals("item_count", new String(p2.getSecond().getQualifier()));
- assertEquals("2", new String(p2.getSecond().getValue()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java
deleted file mode 100644
index 3d54122..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.kv.RowConstants;
-import com.kylinolap.cube.measure.MeasureCodec;
-import com.kylinolap.job.constant.BatchConstants;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CubeReducerTest extends LocalFileMetadataTestCase {
-
- ReduceDriver<Text, Text, Text, Text> reduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- CuboidReducer reducer = new CuboidReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testReducer() throws Exception {
-
- reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
-
- CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
-
- Text key1 = new Text("72010ustech");
- List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
- values1.add(newValueText(codec, "10", "10", "10", 1));
-
- Text key2 = new Text("1tech");
- List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
-
- Text key3 = new Text("0");
- List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
-
- List<Pair<Text, Text>> result = reduceDriver.run();
-
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4));
-
- assertEquals(3, result.size());
-
- assertTrue(result.contains(p1));
- assertTrue(result.contains(p2));
- assertTrue(result.contains(p3));
- }
-
- private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) };
-
- buf.clear();
- codec.encode(values, buf);
-
- Text t = new Text();
- t.set(buf.array(), 0, buf.position());
- return t;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java
deleted file mode 100644
index 14405ec..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package com.kylinolap.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package com.kylinolap.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.FileUtil;
-//import org.apache.hadoop.util.ToolRunner;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import com.kylinolap.metadata.MetadataManager;
-//
-///**
-// * @author xjiang
-// *
-// */
-//public class KeyDistributionJobTest {
-//
-// private Configuration conf;
-//
-// @Before
-// public void setup() throws IOException {
-// conf = new Configuration();
-// conf.set("fs.default.name", "file:///");
-// conf.set("mapred.job.tracker", "local");
-// }
-//
-// @Test
-// public void testJob() throws Exception {
-// final String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
-// final String output = "target/test-output/key_distribution/";
-// final String cubeName = "test_kylin_cube_with_slr";
-// final String metadata = MetadataManager.getMetadataUrlFromEnv();
-//
-// FileUtil.fullyDelete(new File(output));
-//
-// String[] args =
-// { "-input", input, "-cubename", cubeName, "-output", output, "-metadata", metadata,
-// "-columnpercentage", "50", "-splitnumber", "10" };
-// assertEquals("Job failed", 0, ToolRunner.run(new KeyDistributionJob(), args));
-// }
-//
-// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java
deleted file mode 100644
index e7d1748..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java
+++ /dev/null
@@ -1,153 +0,0 @@
-package com.kylinolap.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package com.kylinolap.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.util.List;
-//
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-//import org.apache.hadoop.mrunit.types.Pair;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import com.kylinolap.index.BatchConstants;
-//import com.kylinolap.metadata.MetadataManager;
-//
-///**
-// * @author ysong1
-// *
-// */
-//public class KeyDistributionMapperTest {
-// @SuppressWarnings("rawtypes")
-// MapDriver mapDriver;
-// String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-//
-// @Before
-// public void setUp() {
-// KeyDistributionMapper mapper = new KeyDistributionMapper();
-// mapDriver = MapDriver.newMapDriver(mapper);
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithoutHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_without_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "8");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "26");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2,
-// 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3,
-// 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4,
-// 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5,
-// 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6,
-// 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java
deleted file mode 100644
index 7391191..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author ysong1
- */
-public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- // conf.set("fs.default.name", "file:///");
- // conf.set("mapred.job.tracker", "local");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- // String input =
- // "src/test/resources/data/base_cuboid,src/test/resources/data/6d_cuboid";
- String output = "target/test-output/merged_cuboid";
- String cubeName = "test_kylin_cube_with_slr_ready";
- String jobname = "merge_cuboid";
-
- File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
- baseFolder.delete();
- baseFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
- baseFolder.deleteOnExit();
-
- File sixDFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "6d");
- sixDFolder.delete();
- sixDFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), sixDFolder);
- sixDFolder.deleteOnExit();
-
- FileUtil.fullyDelete(new File(output));
-
- // CubeManager cubeManager =
- // CubeManager.getInstanceFromEnv(this.getTestConfig());
-
- String[] args = { "-input", baseFolder.getAbsolutePath() + "," + sixDFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java
deleted file mode 100644
index ede75b0..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.cube.CubeBuildTypeEnum;
-import com.kylinolap.cube.CubeInstance;
-import com.kylinolap.cube.CubeManager;
-import com.kylinolap.cube.CubeSegment;
-import com.kylinolap.cube.exception.CubeIntegrityException;
-import com.kylinolap.cube.project.ProjectManager;
-import com.kylinolap.dict.Dictionary;
-import com.kylinolap.dict.DictionaryGenerator;
-import com.kylinolap.dict.DictionaryInfo;
-import com.kylinolap.dict.DictionaryManager;
-import com.kylinolap.dict.TrieDictionary;
-import com.kylinolap.dict.lookup.TableSignature;
-import com.kylinolap.job.constant.BatchConstants;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.TblColRef;
-
-/**
- * @author honma
- */
-@SuppressWarnings("rawtypes")
-public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
-
- private static final Logger logger = LoggerFactory.getLogger(MergeCuboidMapperTest.class);
-
- MapDriver<Text, Text, Text, Text> mapDriver;
- CubeManager cubeManager;
- CubeInstance cube;
- DictionaryManager dictionaryManager;
-
- TblColRef lfn;
- TblColRef lsi;
- TblColRef ssc;
-
- private DictionaryInfo makeSharedDict() throws IOException {
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_common_dict");
-
- DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature, "");
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 101, 101, 101 });
- values.add(new byte[] { 102, 102, 102 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(newDictInfo, values);
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- return newDictInfo;
- }
-
- @Before
- public void setUp() throws Exception {
-
- createTestMetadata();
-
- logger.info("The metadataUrl is : " + this.getTestConfig());
-
- MetadataManager.removeInstance(this.getTestConfig());
- CubeManager.removeInstance(this.getTestConfig());
- ProjectManager.removeInstance(this.getTestConfig());
- DictionaryManager.removeInstance(this.getTestConfig());
-
- // hack for distributed cache
- // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to
- // make sure the following mapper could get latest CubeManger
- FileUtils.deleteDirectory(new File("../job/meta"));
-
- MergeCuboidMapper mapper = new MergeCuboidMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
-
- cubeManager = CubeManager.getInstance(this.getTestConfig());
- cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
- dictionaryManager = DictionaryManager.getInstance(getTestConfig());
- lfn = cube.getDescriptor().findColumnRef("TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
- lsi = cube.getDescriptor().findColumnRef("TEST_KYLIN_FACT", "CAL_DT");
- ssc = cube.getDescriptor().findColumnRef("TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
-
- DictionaryInfo sharedDict = makeSharedDict();
-
- boolean isFirstSegment = true;
- for (CubeSegment segment : cube.getSegments()) {
-
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
-
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature, "");
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 97, 97, 97 });
- if (isFirstSegment)
- values.add(new byte[] { 99, 99, 99 });
- else
- values.add(new byte[] { 98, 98, 98 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(newDictInfo, values);
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- segment.putDictResPath(lfn, newDictInfo.getResourcePath());
- segment.putDictResPath(lsi, sharedDict.getResourcePath());
- segment.putDictResPath(ssc, sharedDict.getResourcePath());
-
- // cubeManager.saveResource(segment.getCubeInstance());
- // cubeManager.afterCubeUpdated(segment.getCubeInstance());
- cubeManager.updateCube(cube);
-
- isFirstSegment = false;
- }
-
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void test() throws IOException, ParseException, CubeIntegrityException {
-
- String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments";
-
- List<CubeSegment> newSegments = cubeManager.allocateSegments(cube, CubeBuildTypeEnum.MERGE, 1384240200000L, 1386835200000L);
-
- logger.info("Size of new segments: " + newSegments.size());
-
- CubeSegment newSeg = newSegments.get(0);
- String segmentName = newSeg.getName();
-
- ((TrieDictionary) cubeManager.getDictionary(newSeg, lfn)).dump(System.out);
-
- // hack for distributed cache
- File metaDir = new File("../job/meta");
- FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), metaDir);
-
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL,
- // "../job/meta");
-
- byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 };
- byte[] value = new byte[] { 1, 2, 3 };
- byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 };
- byte[] newvalue = new byte[] { 1, 2, 3 };
-
- mapDriver.withInput(new Text(key), new Text(value));
- mapDriver.withOutput(new Text(newkey), new Text(newvalue));
- mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid"));
-
- mapDriver.runTest();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java
deleted file mode 100644
index a06ddd4..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configuration.IntegerRanges;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.MapContext;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
-import org.apache.hadoop.security.Credentials;
-
-import com.kylinolap.job.constant.BatchConstants;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext {
-
- public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) {
-
- hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
-
- return new WrappedMapper().getMapContext(new MapContext() {
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentKey() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public void write(Object key, Object value) throws IOException, InterruptedException {
- System.out.println("Write -- k:" + key + ", v:" + value);
- if (outKV != null) {
- outKV[0] = key;
- outKV[1] = value;
- }
- }
-
- @Override
- public OutputCommitter getOutputCommitter() {
- throw new NotImplementedException();
- }
-
- @Override
- public TaskAttemptID getTaskAttemptID() {
- throw new NotImplementedException();
- }
-
- @Override
- public void setStatus(String msg) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getStatus() {
- throw new NotImplementedException();
- }
-
- @Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(Enum<?> counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(String groupName, String counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hconf;
- }
-
- @Override
- public Credentials getCredentials() {
- throw new NotImplementedException();
- }
-
- @Override
- public JobID getJobID() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getNumReduceTasks() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path getWorkingDirectory() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJobName() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getSortComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJar() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getGroupingComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getJobSetupCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getProfileEnabled() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getProfileParams() {
- throw new NotImplementedException();
- }
-
- @Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getUser() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getSymlink() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getArchiveClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getFileClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getArchiveTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getFileTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxMapAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxReduceAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public void progress() {
- throw new NotImplementedException();
- }
-
- @Override
- public InputSplit getInputSplit() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getCombinerKeyGroupingComparator() {
- throw new NotImplementedException();
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java
deleted file mode 100644
index 66c23ba..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class NDCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- // conf.set("fs.default.name", "file:///");
- // conf.set("mapred.job.tracker", "local");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob6D() throws Exception {
- String input = "src/test/resources/data/base_cuboid/";
- String output = "target/test-output/6d_cuboid";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "6d_cuboid";
- String level = "1";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
- }
-
- @Test
- public void testJob5D() throws Exception {
- final String input = "src/test/resources/data/6d_cuboid/";
- final String output = "target/test-output/5d_cuboid";
- final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "5d_cuboid";
- String level = "2";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java
deleted file mode 100644
index 96b4aec..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.job.constant.BatchConstants;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
- MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- NDCuboidMapper mapper = new NDCuboidMapper();
- CuboidReducer reducer = new CuboidReducer();
- mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapReduceWithSlr() throws IOException {
-
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
- Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value));
-
- mapReduceDriver.addInput(input1);
-
- List<Pair<Text, Text>> result = mapReduceDriver.run();
-
- assertEquals(4, result.size());
-
- byte[] resultKey = { 0, 0, 0, 0, 0, 0, 1, 127, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
- Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue));
-
- assertTrue(result.contains(output1));
-
- long[] keySet = new long[result.size()];
-
- System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 }));
- for (int i = 0; i < result.size(); i++) {
- byte[] bytes = new byte[result.get(i).getFirst().getLength()];
- System.arraycopy(result.get(i).getFirst().getBytes(), 0, bytes, 0, result.get(i).getFirst().getLength());
- System.out.println(Bytes.toLong(bytes));
- keySet[i] = Bytes.toLong(bytes);
- }
-
- // refer to CuboidSchedulerTest.testGetSpanningCuboid()
- assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet);
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java
deleted file mode 100644
index 9cb1788..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.job.constant.BatchConstants;
-import com.kylinolap.job.hadoop.invertedindex.RandomKeyDistributionMapper;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionMapperTest {
-
- MapDriver<Text, Text, Text, LongWritable> mapDriver;
-
- @Before
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public void setUp() {
- RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @Test
- public void test() throws IOException {
- List<Text> data = new ArrayList<Text>();
- for (int i = 0; i < 1001; i++) {
- data.add(new Text(String.valueOf(i)));
- }
-
- for (Text t : data) {
- mapDriver.addInput(t, new Text("abc"));
- }
-
- mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100");
- List<Pair<Text, LongWritable>> result = mapDriver.run();
- assertEquals(100, result.size());
-
- for (Pair<Text, LongWritable> p : result) {
- System.out.println(p.getFirst());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java
deleted file mode 100644
index 2f18834..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.job.constant.BatchConstants;
-import com.kylinolap.job.hadoop.invertedindex.RandomKeyDistributionReducer;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionReducerTest {
- ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver;
-
- @Before
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public void setUp() {
- RandomKeyDistributionReducer reducer = new RandomKeyDistributionReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @Test
- public void test() throws IOException {
- List<Text> data = new ArrayList<Text>();
- for (int i = 0; i < 1001; i++) {
- data.add(new Text(String.valueOf(i)));
- }
- for (Text t : data) {
- reduceDriver.addInput(t, new ArrayList<LongWritable>());
- }
-
- reduceDriver.getConfiguration().set(BatchConstants.REGION_NUMBER, "2");
- List<Pair<Text, LongWritable>> result = reduceDriver.run();
-
- assertEquals(2, result.size());
-
- for (Pair<Text, LongWritable> p : result) {
- System.out.println(p.getFirst());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java
deleted file mode 100644
index 3ad3124..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- // conf.set("fs.default.name", "file:///");
- // conf.set("mapred.job.tracker", "local");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob() throws Exception {
- String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
- String output = "target/test-output/key_distribution_range/";
- String jobname = "calculate_splits";
- String cubename = "test_kylin_cube_with_slr_ready";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-output", output, "-jobname", jobname, "-cubename", cubename };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new RangeKeyDistributionJob(), args));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java
deleted file mode 100644
index f8a5ff7..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionMapperTest {
-
- @SuppressWarnings("rawtypes")
- MapDriver mapDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() {
- RangeKeyDistributionMapper mapper = new RangeKeyDistributionMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMapperWithoutHeader() throws IOException {
-
- Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey3 = new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey4 = new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey5 = new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey6 = new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey7 = new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-
- mapDriver.addInput(inputKey1, new Text("abc"));
- mapDriver.addInput(inputKey2, new Text("abc"));
- mapDriver.addInput(inputKey3, new Text("abc"));
- mapDriver.addInput(inputKey4, new Text("abc"));
- mapDriver.addInput(inputKey5, new Text("abc"));
- mapDriver.addInput(inputKey6, new Text("abc"));
- mapDriver.addInput(inputKey7, new Text("abc"));
-
- List<Pair<Text, LongWritable>> result = mapDriver.run();
-
- assertEquals(1, result.size());
-
- byte[] key1 = result.get(0).getFirst().getBytes();
- LongWritable value1 = result.get(0).getSecond();
- assertArrayEquals(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1);
- assertEquals(147, value1.get());
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMapperWithHeader() throws IOException {
-
- Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
- Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-
- mapDriver.addInput(inputKey1, new Text("abc"));
- mapDriver.addInput(inputKey2, new Text("abc"));
- mapDriver.addInput(inputKey3, new Text("abc"));
- mapDriver.addInput(inputKey4, new Text("abc"));
- mapDriver.addInput(inputKey5, new Text("abc"));
- mapDriver.addInput(inputKey6, new Text("abc"));
- mapDriver.addInput(inputKey7, new Text("abc"));
-
- List<Pair<Text, LongWritable>> result = mapDriver.run();
-
- assertEquals(1, result.size());
-
- byte[] key1 = result.get(0).getFirst().getBytes();
- LongWritable value1 = result.get(0).getSecond();
- assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1);
- assertEquals(273, value1.get());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java
deleted file mode 100644
index 7e2d2ed..0000000
--- a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.job.constant.BatchConstants;
-
-/**
- * @author ysong1
- *
- */
-public class RangeKeyDistributionReducerTest {
-
- ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() {
- RangeKeyDistributionReducer reducer = new RangeKeyDistributionReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @Test
- public void testReducer() throws IOException {
- Text key1 = new Text(new byte[] { 1 });
- List<LongWritable> values1 = new ArrayList<LongWritable>();
- values1.add(new LongWritable(RangeKeyDistributionReducer.TEN_GIGA_BYTES));
- values1.add(new LongWritable(1));
-
- Text key2 = new Text(new byte[] { 2 });
- List<LongWritable> values2 = new ArrayList<LongWritable>();
- values2.add(new LongWritable(123));
-
- Text key3 = new Text(new byte[] { 3 });
- List<LongWritable> values3 = new ArrayList<LongWritable>();
- values3.add(new LongWritable(RangeKeyDistributionReducer.TEN_GIGA_BYTES));
-
- Text key4 = new Text(new byte[] { 4 });
- List<LongWritable> values4 = new ArrayList<LongWritable>();
- values4.add(new LongWritable(RangeKeyDistributionReducer.TEN_GIGA_BYTES));
-
- Text key5 = new Text(new byte[] { 5 });
- List<LongWritable> values5 = new ArrayList<LongWritable>();
- values5.add(new LongWritable(1));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
- reduceDriver.withInput(key4, values4);
- reduceDriver.withInput(key5, values5);
-
- reduceDriver.getConfiguration().set(BatchConstants.CUBE_CAPACITY, "MEDIUM");
-
- List<Pair<Text, LongWritable>> result = reduceDriver.run();
-
- assertEquals(4, result.size());
-
- byte[] outputKey1 = result.get(0).getFirst().getBytes();
- LongWritable value1 = result.get(0).getSecond();
- assertArrayEquals(new byte[] { 1 }, outputKey1);
- assertEquals(10737418241L, value1.get());
-
- byte[] outputKey2 = result.get(1).getFirst().getBytes();
- LongWritable value2 = result.get(1).getSecond();
- assertArrayEquals(new byte[] { 3 }, outputKey2);
- assertEquals(10737418363L, value2.get());
-
- byte[] outputKey3 = result.get(2).getFirst().getBytes();
- LongWritable value3 = result.get(2).getSecond();
- assertArrayEquals(new byte[] { 4 }, outputKey3);
- assertEquals(10737418240L, value3.get());
-
- byte[] outputKey4 = result.get(3).getFirst().getBytes();
- LongWritable value4 = result.get(3).getSecond();
- assertArrayEquals(new byte[] { 5 }, outputKey4);
- assertEquals(1L, value4.get());
- }
-}