You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/07 15:46:37 UTC
[16/51] [partial] incubator-kylin git commit: migrate repo from
github.com to apache git
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java
new file mode 100644
index 0000000..ed698c4
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidJobTest.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.AbstractKylinTestCase;
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
+
+ private Configuration conf;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new Configuration();
+ // for local runner out-of-memory issue
+ conf.set("mapreduce.task.io.sort.mb", "10");
+
+ createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testJob() throws Exception {
+ String input = "src/test/resources/data/flat_table/";
+ String output = "target/test-output/base_cuboid/";
+ String cubeName = "test_kylin_cube_with_slr_1_new_segment";
+ String segmentName = "20130331080000_20131212080000";
+ String jobname = "base_cuboid_job";
+ String level = "0";
+ FileUtil.fullyDelete(new File(output));
+
+ String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
+ assertEquals("Job failed", 0, ToolRunner.run(conf, new BaseCuboidJob(), args));
+ }
+
+ @Test
+ public void testJobWithBadParas() throws Exception {
+
+ final String input = "src/test/resources/data/flat_table/";
+ final String output = "target/test-output/base_cuboid/";
+ final String metadata = AbstractKylinTestCase.LOCALMETA_TEST_DATA;
+
+ FileUtil.fullyDelete(new File(output));
+
+ String[] args = { "-input", input, "-output", output, "-metadata", metadata };
+ assertEquals(2, ToolRunner.run(conf, new BaseCuboidJob(), args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
new file mode 100644
index 0000000..bab08ec
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Mapper.Context;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * @author yangli9
+ *
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class BaseCuboidMapperPerformanceTest {
+
+ String metadataUrl = "hbase:yadesk00:2181:/hbase-unsecure";
+ String cubeName = "test_kylin_cube_with_slr";
+ Path srcPath = new Path("/download/test_kylin_cube_with_slr_intermediate_table_64mb.seq");
+
+ @Ignore
+ @Test
+ public void test() throws IOException, InterruptedException {
+ Configuration hconf = new Configuration();
+ BaseCuboidMapper mapper = new BaseCuboidMapper();
+ Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
+
+ mapper.setup(context);
+
+ Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
+ Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
+ Text value = new Text();
+
+ while (reader.next(key, value)) {
+ mapper.map(key, value, context);
+ }
+
+ reader.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java
new file mode 100644
index 0000000..95f255f
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/BaseCuboidMapperTest.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.math.BigDecimal;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.kv.RowKeyDecoder;
+import com.kylinolap.cube.measure.MeasureCodec;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.metadata.model.cube.MeasureDesc;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
+
+ MapDriver<Text, Text, Text, Text> mapDriver;
+ String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
+
+ @Before
+ public void setUp() throws Exception {
+ createTestMetadata();
+
+ // hack for distributed cache
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
+
+ BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>();
+ mapDriver = MapDriver.newMapDriver(mapper);
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ }
+
+ @Test
+ public void testMapperWithHeader() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_1_new_segment";
+ String segmentName = "20130331080000_20131212080000";
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+ // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
+ // metadata);
+ mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.331"));
+ List<Pair<Text, Text>> result = mapDriver.run();
+
+ CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig());
+ CubeInstance cube = cubeMgr.getCube(cubeName);
+
+ assertEquals(1, result.size());
+ Text rowkey = result.get(0).getFirst();
+ byte[] key = rowkey.getBytes();
+ byte[] header = Bytes.head(key, 26);
+ byte[] sellerId = Bytes.tail(header, 18);
+ byte[] cuboidId = Bytes.head(header, 8);
+ byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
+
+ RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
+ decoder.decode(key);
+ assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString());
+
+ assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
+ assertEquals(511, Bytes.toLong(cuboidId));
+ assertEquals(22, restKey.length);
+
+ verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33", 1);
+ }
+
+ private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3, long m4) {
+ MeasureCodec codec = new MeasureCodec(measures);
+ Object[] values = new Object[measures.size()];
+ codec.decode(valueBytes, values);
+ assertTrue(new BigDecimal(m1).equals(values[0]));
+ assertTrue(new BigDecimal(m2).equals(values[1]));
+ assertTrue(new BigDecimal(m3).equals(values[2]));
+ assertTrue(m4 == ((LongWritable) values[3]).get());
+ }
+
+ @Test
+ public void testMapperWithNull() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_1_new_segment";
+ String segmentName = "20130331080000_20131212080000";
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+ // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
+ // metadata);
+ mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N\\N"));
+ List<Pair<Text, Text>> result = mapDriver.run();
+
+ CubeManager cubeMgr = CubeManager.getInstance(this.getTestConfig());
+ CubeInstance cube = cubeMgr.getCube(cubeName);
+
+ assertEquals(1, result.size());
+ Text rowkey = result.get(0).getFirst();
+ byte[] key = rowkey.getBytes();
+ byte[] header = Bytes.head(key, 26);
+ byte[] sellerId = Bytes.tail(header, 18);
+ byte[] cuboidId = Bytes.head(header, 8);
+ byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
+
+ RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
+ decoder.decode(key);
+ assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString());
+
+ assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
+ assertEquals(511, Bytes.toLong(cuboidId));
+ assertEquals(22, restKey.length);
+
+ verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", 1L);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java
new file mode 100644
index 0000000..7645222
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/CopySeq.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.hadoop.io.SequenceFile.Writer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * @author yangli9
+ *
+ */
+public class CopySeq {
+
+ public static void main(String[] args) throws IOException {
+ copyTo64MB(args[0], args[1]);
+ }
+
+ public static void copyTo64MB(String src, String dst) throws IOException {
+ Configuration hconf = new Configuration();
+ Path srcPath = new Path(src);
+ Path dstPath = new Path(dst);
+
+ FileSystem fs = FileSystem.get(hconf);
+ long srcSize = fs.getFileStatus(srcPath).getLen();
+ int copyTimes = (int) (67108864 / srcSize); // 64 MB
+ System.out.println("Copy " + copyTimes + " times");
+
+ Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
+ Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
+ Text value = new Text();
+
+ Writer writer = SequenceFile.createWriter(hconf, Writer.file(dstPath), Writer.keyClass(key.getClass()), Writer.valueClass(Text.class), Writer.compression(CompressionType.BLOCK, getLZOCodec(hconf)));
+
+ int count = 0;
+ while (reader.next(key, value)) {
+ for (int i = 0; i < copyTimes; i++) {
+ writer.append(key, value);
+ count++;
+ }
+ }
+
+ System.out.println("Len: " + writer.getLength());
+ System.out.println("Rows: " + count);
+
+ reader.close();
+ writer.close();
+ }
+
+ static CompressionCodec getLZOCodec(Configuration hconf) {
+ CompressionCodecFactory factory = new CompressionCodecFactory(hconf);
+ return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java
new file mode 100644
index 0000000..ffe2fad
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapper2Test.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper.Context;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.kv.RowConstants;
+import com.kylinolap.cube.measure.MeasureCodec;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author yangli9
+ *
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
+
+ String cubeName = "test_kylin_cube_with_slr_ready";
+
+ MeasureCodec codec;
+ ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ Object[] outKV = new Object[2];
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ // hack for distributed cache
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
+ CubeDesc desc = CubeManager.getInstance(this.getTestConfig()).getCube(cubeName).getDescriptor();
+ codec = new MeasureCodec(desc.getMeasures());
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+
+ Configuration hconf = new Configuration();
+ Context context = MockupMapContext.create(hconf, this.getTestConfig().getMetadataUrl(), cubeName, outKV);
+
+ CubeHFileMapper mapper = new CubeHFileMapper();
+ mapper.setup(context);
+
+ Text key = new Text("not important");
+ Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
+
+ mapper.map(key, value, context);
+
+ ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
+ KeyValue outValue = (KeyValue) outKV[1];
+
+ assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(), outKey.getLength()) == 0);
+
+ assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(), outValue.getValueOffset(), outValue.getValueLength()) == 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java
new file mode 100644
index 0000000..0f3aff9
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeHFileMapperTest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.kylinolap.job.constant.BatchConstants;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class CubeHFileMapperTest {
+
+ MapDriver<Text, Text, ImmutableBytesWritable, KeyValue> mapDriver;
+
+ private String cube_name = "FLAT_ITEM_CUBE";
+
+ @Before
+ public void setUp() {
+ CubeHFileMapper mapper = new CubeHFileMapper();
+ mapDriver = MapDriver.newMapDriver(mapper);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ @Ignore
+ public void testMapper2() throws IOException {
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name);
+
+ mapDriver.addInput(new Text("52010tech"), new Text("35.432"));
+
+ List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run();
+
+ assertEquals(2, result.size());
+
+ byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 };
+ ImmutableBytesWritable key = new ImmutableBytesWritable(bytes);
+
+ Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0);
+ Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1);
+
+ assertEquals(key, p1.getFirst());
+ assertEquals("cf1", new String(p1.getSecond().getFamily()));
+ assertEquals("usd_amt", new String(p1.getSecond().getQualifier()));
+ assertEquals("35.43", new String(p1.getSecond().getValue()));
+
+ assertEquals(key, p2.getFirst());
+ assertEquals("cf1", new String(p2.getSecond().getFamily()));
+ assertEquals("item_count", new String(p2.getSecond().getQualifier()));
+ assertEquals("2", new String(p2.getSecond().getValue()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java
new file mode 100644
index 0000000..3d54122
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/CubeReducerTest.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.kv.RowConstants;
+import com.kylinolap.cube.measure.MeasureCodec;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.metadata.model.cube.CubeDesc;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class CubeReducerTest extends LocalFileMetadataTestCase {
+
+ ReduceDriver<Text, Text, Text, Text> reduceDriver;
+ String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
+
+ ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+
+ @Before
+ public void setUp() throws Exception {
+ createTestMetadata();
+
+ // hack for distributed cache
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
+
+ CuboidReducer reducer = new CuboidReducer();
+ reduceDriver = ReduceDriver.newReduceDriver(reducer);
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ }
+
+ @Test
+ public void testReducer() throws Exception {
+
+ reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
+
+ CubeDesc cubeDesc = CubeManager.getInstance(this.getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
+ MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
+
+ Text key1 = new Text("72010ustech");
+ List<Text> values1 = new ArrayList<Text>();
+ values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
+ values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
+ values1.add(newValueText(codec, "10", "10", "10", 1));
+
+ Text key2 = new Text("1tech");
+ List<Text> values2 = new ArrayList<Text>();
+ values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
+ values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
+
+ Text key3 = new Text("0");
+ List<Text> values3 = new ArrayList<Text>();
+ values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4));
+
+ reduceDriver.withInput(key1, values1);
+ reduceDriver.withInput(key2, values2);
+ reduceDriver.withInput(key3, values3);
+
+ List<Pair<Text, Text>> result = reduceDriver.run();
+
+ Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3));
+ Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2));
+ Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4));
+
+ assertEquals(3, result.size());
+
+ assertTrue(result.contains(p1));
+ assertTrue(result.contains(p2));
+ assertTrue(result.contains(p3));
+ }
+
+ private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) {
+ Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) };
+
+ buf.clear();
+ codec.encode(values, buf);
+
+ Text t = new Text();
+ t.set(buf.array(), 0, buf.position());
+ return t;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java
new file mode 100644
index 0000000..14405ec
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionJobTest.java
@@ -0,0 +1,64 @@
+package com.kylinolap.job.hadoop.cube;
+
+///*
+// * Copyright 2013-2014 eBay Software Foundation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package com.kylinolap.index.cube;
+//
+//import static org.junit.Assert.*;
+//
+//import java.io.File;
+//import java.io.IOException;
+//
+//import org.apache.hadoop.conf.Configuration;
+//import org.apache.hadoop.fs.FileUtil;
+//import org.apache.hadoop.util.ToolRunner;
+//import org.junit.Before;
+//import org.junit.Test;
+//
+//import com.kylinolap.metadata.MetadataManager;
+//
+///**
+// * @author xjiang
+// *
+// */
+//public class KeyDistributionJobTest {
+//
+// private Configuration conf;
+//
+// @Before
+// public void setup() throws IOException {
+// conf = new Configuration();
+// conf.set("fs.default.name", "file:///");
+// conf.set("mapred.job.tracker", "local");
+// }
+//
+// @Test
+// public void testJob() throws Exception {
+// final String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
+// final String output = "target/test-output/key_distribution/";
+// final String cubeName = "test_kylin_cube_with_slr";
+// final String metadata = MetadataManager.getMetadataUrlFromEnv();
+//
+// FileUtil.fullyDelete(new File(output));
+//
+// String[] args =
+// { "-input", input, "-cubename", cubeName, "-output", output, "-metadata", metadata,
+// "-columnpercentage", "50", "-splitnumber", "10" };
+// assertEquals("Job failed", 0, ToolRunner.run(new KeyDistributionJob(), args));
+// }
+//
+// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java
new file mode 100644
index 0000000..e7d1748
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/KeyDistributionMapperTest.java
@@ -0,0 +1,153 @@
+package com.kylinolap.job.hadoop.cube;
+
+///*
+// * Copyright 2013-2014 eBay Software Foundation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package com.kylinolap.index.cube;
+//
+//import static org.junit.Assert.*;
+//
+//import java.io.File;
+//import java.io.IOException;
+//import java.util.List;
+//
+//import org.apache.hadoop.io.LongWritable;
+//import org.apache.hadoop.io.Text;
+//import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+//import org.apache.hadoop.mrunit.types.Pair;
+//import org.junit.Before;
+//import org.junit.Test;
+//
+//import com.kylinolap.index.BatchConstants;
+//import com.kylinolap.metadata.MetadataManager;
+//
+///**
+// * @author ysong1
+// *
+// */
+//public class KeyDistributionMapperTest {
+// @SuppressWarnings("rawtypes")
+// MapDriver mapDriver;
+// String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
+//
+// @Before
+// public void setUp() {
+// KeyDistributionMapper mapper = new KeyDistributionMapper();
+// mapDriver = MapDriver.newMapDriver(mapper);
+// }
+//
+// @SuppressWarnings("unchecked")
+// @Test
+// public void testMapperWithoutHeader() throws IOException {
+// String matadata = MetadataManager.getMetadataUrlFromEnv();
+// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_without_slr");
+// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
+// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
+// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "8");
+//
+// Text inputKey1 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey2 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey3 =
+// new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey4 =
+// new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey5 =
+// new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey6 =
+// new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey7 =
+// new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+//
+// mapDriver.addInput(inputKey1, new Text("abc"));
+// mapDriver.addInput(inputKey2, new Text("abc"));
+// mapDriver.addInput(inputKey3, new Text("abc"));
+// mapDriver.addInput(inputKey4, new Text("abc"));
+// mapDriver.addInput(inputKey5, new Text("abc"));
+// mapDriver.addInput(inputKey6, new Text("abc"));
+// mapDriver.addInput(inputKey7, new Text("abc"));
+//
+// List<Pair<Text, LongWritable>> result = mapDriver.run();
+//
+// assertEquals(7, result.size());
+//
+// byte[] key1 = result.get(0).getFirst().getBytes();
+// LongWritable value1 = result.get(0).getSecond();
+// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11 }, key1);
+// assertEquals(2, value1.get());
+//
+// byte[] key7 = result.get(6).getFirst().getBytes();
+// LongWritable value7 = result.get(6).getSecond();
+// assertArrayEquals(new byte[] { 0 }, key7);
+// assertEquals(7, value7.get());
+// }
+//
+// @SuppressWarnings("unchecked")
+// @Test
+// public void testMapperWithHeader() throws IOException {
+// String matadata = MetadataManager.getMetadataUrlFromEnv();
+// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr");
+// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
+// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
+// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "26");
+//
+// Text inputKey1 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
+// 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey2 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
+// 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey3 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2,
+// 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey4 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3,
+// 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey5 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4,
+// 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey6 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5,
+// 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+// Text inputKey7 =
+// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6,
+// 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+//
+// mapDriver.addInput(inputKey1, new Text("abc"));
+// mapDriver.addInput(inputKey2, new Text("abc"));
+// mapDriver.addInput(inputKey3, new Text("abc"));
+// mapDriver.addInput(inputKey4, new Text("abc"));
+// mapDriver.addInput(inputKey5, new Text("abc"));
+// mapDriver.addInput(inputKey6, new Text("abc"));
+// mapDriver.addInput(inputKey7, new Text("abc"));
+//
+// List<Pair<Text, LongWritable>> result = mapDriver.run();
+//
+// assertEquals(7, result.size());
+//
+// byte[] key1 = result.get(0).getFirst().getBytes();
+// LongWritable value1 = result.get(0).getSecond();
+// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
+// 0, 0, 0, 0, 0, 127, 11 }, key1);
+// assertEquals(2, value1.get());
+//
+// byte[] key7 = result.get(6).getFirst().getBytes();
+// LongWritable value7 = result.get(6).getSecond();
+// assertArrayEquals(new byte[] { 0 }, key7);
+// assertEquals(7, value7.get());
+// }
+// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java
new file mode 100644
index 0000000..7391191
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidJobTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+
+/**
+ * @author ysong1
+ */
+public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
+
+ private Configuration conf;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new Configuration();
+ // conf.set("fs.default.name", "file:///");
+ // conf.set("mapred.job.tracker", "local");
+
+ // for local runner out-of-memory issue
+ conf.set("mapreduce.task.io.sort.mb", "10");
+ createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ // String input =
+ // "src/test/resources/data/base_cuboid,src/test/resources/data/6d_cuboid";
+ String output = "target/test-output/merged_cuboid";
+ String cubeName = "test_kylin_cube_with_slr_ready";
+ String jobname = "merge_cuboid";
+
+ File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
+ baseFolder.delete();
+ baseFolder.mkdir();
+ FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
+ baseFolder.deleteOnExit();
+
+ File sixDFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "6d");
+ sixDFolder.delete();
+ sixDFolder.mkdir();
+ FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), sixDFolder);
+ sixDFolder.deleteOnExit();
+
+ FileUtil.fullyDelete(new File(output));
+
+ // CubeManager cubeManager =
+ // CubeManager.getInstanceFromEnv(this.getTestConfig());
+
+ String[] args = { "-input", baseFolder.getAbsolutePath() + "," + sixDFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
+ assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java
new file mode 100644
index 0000000..ede75b0
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.cube.CubeBuildTypeEnum;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.CubeManager;
+import com.kylinolap.cube.CubeSegment;
+import com.kylinolap.cube.exception.CubeIntegrityException;
+import com.kylinolap.cube.project.ProjectManager;
+import com.kylinolap.dict.Dictionary;
+import com.kylinolap.dict.DictionaryGenerator;
+import com.kylinolap.dict.DictionaryInfo;
+import com.kylinolap.dict.DictionaryManager;
+import com.kylinolap.dict.TrieDictionary;
+import com.kylinolap.dict.lookup.TableSignature;
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.metadata.MetadataManager;
+import com.kylinolap.metadata.model.cube.TblColRef;
+
+/**
+ * @author honma
+ */
+@SuppressWarnings("rawtypes")
+public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeCuboidMapperTest.class);
+
+ MapDriver<Text, Text, Text, Text> mapDriver;
+ CubeManager cubeManager;
+ CubeInstance cube;
+ DictionaryManager dictionaryManager;
+
+ TblColRef lfn;
+ TblColRef lsi;
+ TblColRef ssc;
+
+ private DictionaryInfo makeSharedDict() throws IOException {
+ TableSignature signature = new TableSignature();
+ signature.setSize(100);
+ signature.setLastModifiedTime(System.currentTimeMillis());
+ signature.setPath("fake_common_dict");
+
+ DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature, "");
+
+ List<byte[]> values = new ArrayList<byte[]>();
+ values.add(new byte[] { 101, 101, 101 });
+ values.add(new byte[] { 102, 102, 102 });
+ Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(newDictInfo, values);
+ dictionaryManager.trySaveNewDict(dict, newDictInfo);
+ ((TrieDictionary) dict).dump(System.out);
+
+ return newDictInfo;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+
+ createTestMetadata();
+
+ logger.info("The metadataUrl is : " + this.getTestConfig());
+
+ MetadataManager.removeInstance(this.getTestConfig());
+ CubeManager.removeInstance(this.getTestConfig());
+ ProjectManager.removeInstance(this.getTestConfig());
+ DictionaryManager.removeInstance(this.getTestConfig());
+
+ // hack for distributed cache
+ // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to
+ // make sure the following mapper could get latest CubeManger
+ FileUtils.deleteDirectory(new File("../job/meta"));
+
+ MergeCuboidMapper mapper = new MergeCuboidMapper();
+ mapDriver = MapDriver.newMapDriver(mapper);
+
+ cubeManager = CubeManager.getInstance(this.getTestConfig());
+ cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
+ dictionaryManager = DictionaryManager.getInstance(getTestConfig());
+ lfn = cube.getDescriptor().findColumnRef("TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
+ lsi = cube.getDescriptor().findColumnRef("TEST_KYLIN_FACT", "CAL_DT");
+ ssc = cube.getDescriptor().findColumnRef("TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
+
+ DictionaryInfo sharedDict = makeSharedDict();
+
+ boolean isFirstSegment = true;
+ for (CubeSegment segment : cube.getSegments()) {
+
+ TableSignature signature = new TableSignature();
+ signature.setSize(100);
+ signature.setLastModifiedTime(System.currentTimeMillis());
+ signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
+
+ DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature, "");
+
+ List<byte[]> values = new ArrayList<byte[]>();
+ values.add(new byte[] { 97, 97, 97 });
+ if (isFirstSegment)
+ values.add(new byte[] { 99, 99, 99 });
+ else
+ values.add(new byte[] { 98, 98, 98 });
+ Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(newDictInfo, values);
+ dictionaryManager.trySaveNewDict(dict, newDictInfo);
+ ((TrieDictionary) dict).dump(System.out);
+
+ segment.putDictResPath(lfn, newDictInfo.getResourcePath());
+ segment.putDictResPath(lsi, sharedDict.getResourcePath());
+ segment.putDictResPath(ssc, sharedDict.getResourcePath());
+
+ // cubeManager.saveResource(segment.getCubeInstance());
+ // cubeManager.afterCubeUpdated(segment.getCubeInstance());
+ cubeManager.updateCube(cube);
+
+ isFirstSegment = false;
+ }
+
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ }
+
+ @Test
+ public void test() throws IOException, ParseException, CubeIntegrityException {
+
+ String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments";
+
+ List<CubeSegment> newSegments = cubeManager.allocateSegments(cube, CubeBuildTypeEnum.MERGE, 1384240200000L, 1386835200000L);
+
+ logger.info("Size of new segments: " + newSegments.size());
+
+ CubeSegment newSeg = newSegments.get(0);
+ String segmentName = newSeg.getName();
+
+ ((TrieDictionary) cubeManager.getDictionary(newSeg, lfn)).dump(System.out);
+
+ // hack for distributed cache
+ File metaDir = new File("../job/meta");
+ FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), metaDir);
+
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+ // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL,
+ // "../job/meta");
+
+ byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 };
+ byte[] value = new byte[] { 1, 2, 3 };
+ byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 };
+ byte[] newvalue = new byte[] { 1, 2, 3 };
+
+ mapDriver.withInput(new Text(key), new Text(value));
+ mapDriver.withOutput(new Text(newkey), new Text(newvalue));
+ mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid"));
+
+ mapDriver.runTest();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java
new file mode 100644
index 0000000..a06ddd4
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/MockupMapContext.java
@@ -0,0 +1,312 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MapContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Mapper.Context;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+import org.apache.hadoop.security.Credentials;
+
+import com.kylinolap.job.constant.BatchConstants;
+
+/**
+ * @author yangli9
+ *
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class MockupMapContext {
+
+ public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) {
+
+ hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
+
+ return new WrappedMapper().getMapContext(new MapContext() {
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object getCurrentKey() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object getCurrentValue() throws IOException, InterruptedException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void write(Object key, Object value) throws IOException, InterruptedException {
+ System.out.println("Write -- k:" + key + ", v:" + value);
+ if (outKV != null) {
+ outKV[0] = key;
+ outKV[1] = value;
+ }
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public TaskAttemptID getTaskAttemptID() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void setStatus(String msg) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getStatus() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public float getProgress() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Counter getCounter(Enum<?> counterName) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Counter getCounter(String groupName, String counterName) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Configuration getConfiguration() {
+ return hconf;
+ }
+
+ @Override
+ public Credentials getCredentials() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public JobID getJobID() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getNumReduceTasks() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path getWorkingDirectory() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getOutputKeyClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getOutputValueClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getMapOutputKeyClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<?> getMapOutputValueClass() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJobName() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getSortComparator() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getJar() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getJobSetupCleanupNeeded() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getTaskCleanupNeeded() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getProfileEnabled() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getProfileParams() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public IntegerRanges getProfileTaskRange(boolean isMap) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String getUser() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public boolean getSymlink() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getArchiveClassPaths() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public URI[] getCacheArchives() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public URI[] getCacheFiles() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getLocalCacheArchives() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getLocalCacheFiles() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Path[] getFileClassPaths() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String[] getArchiveTimestamps() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public String[] getFileTimestamps() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getMaxMapAttempts() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getMaxReduceAttempts() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void progress() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public InputSplit getInputSplit() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public RawComparator<?> getCombinerKeyGroupingComparator() {
+ throw new NotImplementedException();
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java
new file mode 100644
index 0000000..66c23ba
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidJobTest.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class NDCuboidJobTest extends LocalFileMetadataTestCase {
+
+ private Configuration conf;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new Configuration();
+ // conf.set("fs.default.name", "file:///");
+ // conf.set("mapred.job.tracker", "local");
+
+ // for local runner out-of-memory issue
+ conf.set("mapreduce.task.io.sort.mb", "10");
+
+ createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testJob6D() throws Exception {
+ String input = "src/test/resources/data/base_cuboid/";
+ String output = "target/test-output/6d_cuboid";
+ String cubeName = "test_kylin_cube_with_slr_1_new_segment";
+ String segmentName = "20130331080000_20131212080000";
+ String jobname = "6d_cuboid";
+ String level = "1";
+
+ FileUtil.fullyDelete(new File(output));
+
+ String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
+ assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
+ }
+
+ @Test
+ public void testJob5D() throws Exception {
+ final String input = "src/test/resources/data/6d_cuboid/";
+ final String output = "target/test-output/5d_cuboid";
+ final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
+ String segmentName = "20130331080000_20131212080000";
+ String jobname = "5d_cuboid";
+ String level = "2";
+
+ FileUtil.fullyDelete(new File(output));
+
+ String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
+ assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java
new file mode 100644
index 0000000..96b4aec
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/NDCuboidMapperTest.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+import com.kylinolap.job.constant.BatchConstants;
+
+/**
+ * @author George Song (ysong1)
+ *
+ */
+public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
+ MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver;
+ String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
+
+ @Before
+ public void setUp() throws Exception {
+ createTestMetadata();
+
+ // hack for distributed cache
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ FileUtils.copyDirectory(new File(this.getTestConfig().getMetadataUrl()), new File("../job/meta"));
+
+ NDCuboidMapper mapper = new NDCuboidMapper();
+ CuboidReducer reducer = new CuboidReducer();
+ mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ FileUtils.deleteDirectory(new File("../job/meta"));
+ }
+
+ @Test
+ public void testMapReduceWithSlr() throws IOException {
+
+ String cubeName = "test_kylin_cube_with_slr_1_new_segment";
+ String segmentName = "20130331080000_20131212080000";
+ mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
+ mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
+
+ byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
+ byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
+ Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value));
+
+ mapReduceDriver.addInput(input1);
+
+ List<Pair<Text, Text>> result = mapReduceDriver.run();
+
+ assertEquals(4, result.size());
+
+ byte[] resultKey = { 0, 0, 0, 0, 0, 0, 1, 127, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
+ byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
+ Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue));
+
+ assertTrue(result.contains(output1));
+
+ long[] keySet = new long[result.size()];
+
+ System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 }));
+ for (int i = 0; i < result.size(); i++) {
+ byte[] bytes = new byte[result.get(i).getFirst().getLength()];
+ System.arraycopy(result.get(i).getFirst().getBytes(), 0, bytes, 0, result.get(i).getFirst().getLength());
+ System.out.println(Bytes.toLong(bytes));
+ keySet[i] = Bytes.toLong(bytes);
+ }
+
+ // refer to CuboidSchedulerTest.testGetSpanningCuboid()
+ assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet);
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java
new file mode 100644
index 0000000..9cb1788
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionMapperTest.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.invertedindex.RandomKeyDistributionMapper;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RandomKeyDistributionMapperTest {
+
+ MapDriver<Text, Text, Text, LongWritable> mapDriver;
+
+ @Before
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public void setUp() {
+ RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper();
+ mapDriver = MapDriver.newMapDriver(mapper);
+ }
+
+ @Test
+ public void test() throws IOException {
+ List<Text> data = new ArrayList<Text>();
+ for (int i = 0; i < 1001; i++) {
+ data.add(new Text(String.valueOf(i)));
+ }
+
+ for (Text t : data) {
+ mapDriver.addInput(t, new Text("abc"));
+ }
+
+ mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100");
+ List<Pair<Text, LongWritable>> result = mapDriver.run();
+ assertEquals(100, result.size());
+
+ for (Pair<Text, LongWritable> p : result) {
+ System.out.println(p.getFirst());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java
new file mode 100644
index 0000000..2f18834
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/RandomKeyDistributionReducerTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.job.constant.BatchConstants;
+import com.kylinolap.job.hadoop.invertedindex.RandomKeyDistributionReducer;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RandomKeyDistributionReducerTest {
+ ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver;
+
+ @Before
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public void setUp() {
+ RandomKeyDistributionReducer reducer = new RandomKeyDistributionReducer();
+ reduceDriver = ReduceDriver.newReduceDriver(reducer);
+ }
+
+ @Test
+ public void test() throws IOException {
+ List<Text> data = new ArrayList<Text>();
+ for (int i = 0; i < 1001; i++) {
+ data.add(new Text(String.valueOf(i)));
+ }
+ for (Text t : data) {
+ reduceDriver.addInput(t, new ArrayList<LongWritable>());
+ }
+
+ reduceDriver.getConfiguration().set(BatchConstants.REGION_NUMBER, "2");
+ List<Pair<Text, LongWritable>> result = reduceDriver.run();
+
+ assertEquals(2, result.size());
+
+ for (Pair<Text, LongWritable> p : result) {
+ System.out.println(p.getFirst());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java
new file mode 100644
index 0000000..3ad3124
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionJobTest.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.common.util.LocalFileMetadataTestCase;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RangeKeyDistributionJobTest extends LocalFileMetadataTestCase {
+
+ private Configuration conf;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new Configuration();
+ // conf.set("fs.default.name", "file:///");
+ // conf.set("mapred.job.tracker", "local");
+
+ // for local runner out-of-memory issue
+ conf.set("mapreduce.task.io.sort.mb", "10");
+ createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testJob() throws Exception {
+ String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
+ String output = "target/test-output/key_distribution_range/";
+ String jobname = "calculate_splits";
+ String cubename = "test_kylin_cube_with_slr_ready";
+
+ FileUtil.fullyDelete(new File(output));
+
+ String[] args = { "-input", input, "-output", output, "-jobname", jobname, "-cubename", cubename };
+ assertEquals("Job failed", 0, ToolRunner.run(conf, new RangeKeyDistributionJob(), args));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java
new file mode 100644
index 0000000..f8a5ff7
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionMapperTest.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RangeKeyDistributionMapperTest {
+
+ @SuppressWarnings("rawtypes")
+ MapDriver mapDriver;
+ String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
+
+ @Before
+ public void setUp() {
+ RangeKeyDistributionMapper mapper = new RangeKeyDistributionMapper();
+ mapDriver = MapDriver.newMapDriver(mapper);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapperWithoutHeader() throws IOException {
+
+ Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey3 = new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey4 = new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey5 = new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey6 = new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey7 = new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+
+ mapDriver.addInput(inputKey1, new Text("abc"));
+ mapDriver.addInput(inputKey2, new Text("abc"));
+ mapDriver.addInput(inputKey3, new Text("abc"));
+ mapDriver.addInput(inputKey4, new Text("abc"));
+ mapDriver.addInput(inputKey5, new Text("abc"));
+ mapDriver.addInput(inputKey6, new Text("abc"));
+ mapDriver.addInput(inputKey7, new Text("abc"));
+
+ List<Pair<Text, LongWritable>> result = mapDriver.run();
+
+ assertEquals(1, result.size());
+
+ byte[] key1 = result.get(0).getFirst().getBytes();
+ LongWritable value1 = result.get(0).getSecond();
+ assertArrayEquals(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1);
+ assertEquals(147, value1.get());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapperWithHeader() throws IOException {
+
+ Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+ Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
+
+ mapDriver.addInput(inputKey1, new Text("abc"));
+ mapDriver.addInput(inputKey2, new Text("abc"));
+ mapDriver.addInput(inputKey3, new Text("abc"));
+ mapDriver.addInput(inputKey4, new Text("abc"));
+ mapDriver.addInput(inputKey5, new Text("abc"));
+ mapDriver.addInput(inputKey6, new Text("abc"));
+ mapDriver.addInput(inputKey7, new Text("abc"));
+
+ List<Pair<Text, LongWritable>> result = mapDriver.run();
+
+ assertEquals(1, result.size());
+
+ byte[] key1 = result.get(0).getFirst().getBytes();
+ LongWritable value1 = result.get(0).getSecond();
+ assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1);
+ assertEquals(273, value1.get());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b631f92/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java
new file mode 100644
index 0000000..7e2d2ed
--- /dev/null
+++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/RangeKeyDistributionReducerTest.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2013-2014 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kylinolap.job.hadoop.cube;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.kylinolap.job.constant.BatchConstants;
+
+/**
+ * @author ysong1
+ *
+ */
+public class RangeKeyDistributionReducerTest {
+
+ ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver;
+ String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
+
+ @Before
+ public void setUp() {
+ RangeKeyDistributionReducer reducer = new RangeKeyDistributionReducer();
+ reduceDriver = ReduceDriver.newReduceDriver(reducer);
+ }
+
+ @Test
+ public void testReducer() throws IOException {
+ Text key1 = new Text(new byte[] { 1 });
+ List<LongWritable> values1 = new ArrayList<LongWritable>();
+ values1.add(new LongWritable(RangeKeyDistributionReducer.TEN_GIGA_BYTES));
+ values1.add(new LongWritable(1));
+
+ Text key2 = new Text(new byte[] { 2 });
+ List<LongWritable> values2 = new ArrayList<LongWritable>();
+ values2.add(new LongWritable(123));
+
+ Text key3 = new Text(new byte[] { 3 });
+ List<LongWritable> values3 = new ArrayList<LongWritable>();
+ values3.add(new LongWritable(RangeKeyDistributionReducer.TEN_GIGA_BYTES));
+
+ Text key4 = new Text(new byte[] { 4 });
+ List<LongWritable> values4 = new ArrayList<LongWritable>();
+ values4.add(new LongWritable(RangeKeyDistributionReducer.TEN_GIGA_BYTES));
+
+ Text key5 = new Text(new byte[] { 5 });
+ List<LongWritable> values5 = new ArrayList<LongWritable>();
+ values5.add(new LongWritable(1));
+
+ reduceDriver.withInput(key1, values1);
+ reduceDriver.withInput(key2, values2);
+ reduceDriver.withInput(key3, values3);
+ reduceDriver.withInput(key4, values4);
+ reduceDriver.withInput(key5, values5);
+
+ reduceDriver.getConfiguration().set(BatchConstants.CUBE_CAPACITY, "MEDIUM");
+
+ List<Pair<Text, LongWritable>> result = reduceDriver.run();
+
+ assertEquals(4, result.size());
+
+ byte[] outputKey1 = result.get(0).getFirst().getBytes();
+ LongWritable value1 = result.get(0).getSecond();
+ assertArrayEquals(new byte[] { 1 }, outputKey1);
+ assertEquals(10737418241L, value1.get());
+
+ byte[] outputKey2 = result.get(1).getFirst().getBytes();
+ LongWritable value2 = result.get(1).getSecond();
+ assertArrayEquals(new byte[] { 3 }, outputKey2);
+ assertEquals(10737418363L, value2.get());
+
+ byte[] outputKey3 = result.get(2).getFirst().getBytes();
+ LongWritable value3 = result.get(2).getSecond();
+ assertArrayEquals(new byte[] { 4 }, outputKey3);
+ assertEquals(10737418240L, value3.get());
+
+ byte[] outputKey4 = result.get(3).getFirst().getBytes();
+ LongWritable value4 = result.get(3).getSecond();
+ assertArrayEquals(new byte[] { 5 }, outputKey4);
+ assertEquals(1L, value4.get());
+ }
+}