You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/09/06 09:39:11 UTC
[05/50] [abbrv] incubator-kylin git commit: KYLIN-934 update test
case data
KYLIN-934 update test case data
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/34f37260
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/34f37260
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/34f37260
Branch: refs/heads/master
Commit: 34f37260a3d3b6e6852ac8f4fe352a590676619e
Parents: a60ec16
Author: honma <ho...@ebay.com>
Authored: Tue Aug 11 14:16:43 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Aug 11 14:16:58 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cube/BaseCuboidJobTest.java | 4 ---
.../job/hadoop/cube/BaseCuboidMapperTest.java | 24 ++++++++---------
.../kylin/job/hadoop/cube/CubeReducerTest.java | 27 ++++++++-----------
.../job/hadoop/cube/MergeCuboidJobTest.java | 15 +++++------
.../kylin/job/hadoop/cube/NDCuboidJobTest.java | 18 +++++--------
.../job/hadoop/cube/NDCuboidMapperTest.java | 9 ++-----
.../test/resources/data/6d_cuboid/part-r-00000 | Bin 3279797 -> 0 bytes
.../test/resources/data/8d_cuboid/part-r-00000 | Bin 0 -> 1476517 bytes
.../resources/data/base_cuboid/part-r-00000 | Bin 870735 -> 394644 bytes
job/src/test/resources/data/flat_table/000000_0 | Bin 249140 -> 110778 bytes
10 files changed, 37 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
index 771849f..949040b 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
@@ -30,10 +30,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * @author George Song (ysong1)
- *
- */
public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
private Configuration conf;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
index dde0afa..c7b5695 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java
@@ -26,6 +26,7 @@ import java.math.BigDecimal;
import java.util.List;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.types.Pair;
@@ -41,14 +42,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * @author George Song (ysong1)
- *
- */
public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
MapDriver<Text, Text, Text, Text> mapDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
@Before
public void setUp() throws Exception {
@@ -76,7 +72,7 @@ public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
// metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33"));
+ mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.3322"));
List<Pair<Text, Text>> result = mapDriver.run();
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
@@ -98,16 +94,18 @@ public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
assertEquals(511, Bytes.toLong(cuboidId));
assertEquals(22, restKey.length);
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33");
+ verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33", "1", "22");
}
- private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3) {
+ private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String... valueStr) {
MeasureCodec codec = new MeasureCodec(measures);
Object[] values = new Object[measures.size()];
codec.decode(valueBytes, values);
- assertTrue(new BigDecimal(m1).equals(values[0]));
- assertTrue(new BigDecimal(m2).equals(values[1]));
- assertTrue(new BigDecimal(m3).equals(values[2]));
+ assertTrue(new BigDecimal(valueStr[0]).equals(values[0]));
+ assertTrue(new BigDecimal(valueStr[1]).equals(values[1]));
+ assertTrue(new BigDecimal(valueStr[2]).equals(values[2]));
+ assertTrue(new LongWritable(Long.valueOf(valueStr[3])).equals(values[3]));
+ assertTrue(new LongWritable(Long.valueOf(valueStr[4])).equals(values[4]));
}
@Test
@@ -118,7 +116,7 @@ public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
// metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N"));
+ mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N22"));
List<Pair<Text, Text>> result = mapDriver.run();
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
@@ -140,6 +138,6 @@ public class BaseCuboidMapperTest extends LocalFileMetadataTestCase {
assertEquals(511, Bytes.toLong(cuboidId));
assertEquals(22, restKey.length);
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0");
+ verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", "1","22");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
index 96b4a5f..5bcc088 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
@@ -42,14 +42,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * @author George Song (ysong1)
- *
- */
public class CubeReducerTest extends LocalFileMetadataTestCase {
ReduceDriver<Text, Text, Text, Text> reduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
@@ -81,18 +76,18 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
Text key1 = new Text("72010ustech");
List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
- values1.add(newValueText(codec, "10", "10", "10", 1));
+ values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1,22));
+ values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1,23));
+ values1.add(newValueText(codec, "10", "10", "10", 1,24));
Text key2 = new Text("1tech");
List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
+ values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1,12));
+ values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1,13));
Text key3 = new Text("0");
List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4));
+ values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4,11));
reduceDriver.withInput(key1, values1);
reduceDriver.withInput(key2, values2);
@@ -100,9 +95,9 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
List<Pair<Text, Text>> result = reduceDriver.run();
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4));
+ Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3,69));
+ Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2,25));
+ Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4,11));
assertEquals(3, result.size());
@@ -111,8 +106,8 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
assertTrue(result.contains(p3));
}
- private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count) };
+ private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count,int itemcount) {
+ Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count),new LongWritable(itemcount) };
buf.clear();
codec.encode(values, buf);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
index 182310b..4b853f3 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
@@ -31,9 +31,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * @author ysong1
- */
public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
private Configuration conf;
@@ -68,18 +65,18 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
baseFolder.deleteOnExit();
- File sixDFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "6d");
- sixDFolder.delete();
- sixDFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), sixDFolder);
- sixDFolder.deleteOnExit();
+ File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d");
+ eightFolder.delete();
+ eightFolder.mkdir();
+ FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder);
+ eightFolder.deleteOnExit();
FileUtil.fullyDelete(new File(output));
// CubeManager cubeManager =
// CubeManager.getInstanceFromEnv(getTestConfig());
- String[] args = { "-input", baseFolder.getAbsolutePath() + "," + sixDFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
+ String[] args = { "-input", baseFolder.getAbsolutePath() + "," + eightFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
index e77a820..ede23bb 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
@@ -30,10 +30,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * @author George Song (ysong1)
- *
- */
public class NDCuboidJobTest extends LocalFileMetadataTestCase {
private Configuration conf;
@@ -56,12 +52,12 @@ public class NDCuboidJobTest extends LocalFileMetadataTestCase {
}
@Test
- public void testJob6D() throws Exception {
+ public void testJob8D() throws Exception {
String input = "src/test/resources/data/base_cuboid/";
- String output = "target/test-output/6d_cuboid";
+ String output = "target/test-output/8d_cuboid";
String cubeName = "test_kylin_cube_with_slr_1_new_segment";
String segmentName = "20130331080000_20131212080000";
- String jobname = "6d_cuboid";
+ String jobname = "8d_cuboid";
String level = "1";
FileUtil.fullyDelete(new File(output));
@@ -71,12 +67,12 @@ public class NDCuboidJobTest extends LocalFileMetadataTestCase {
}
@Test
- public void testJob5D() throws Exception {
- final String input = "src/test/resources/data/6d_cuboid/";
- final String output = "target/test-output/5d_cuboid";
+ public void testJob7D() throws Exception {
+ final String input = "src/test/resources/data/8d_cuboid/";
+ final String output = "target/test-output/7d_cuboid";
final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
String segmentName = "20130331080000_20131212080000";
- String jobname = "5d_cuboid";
+ String jobname = "7d_cuboid";
String level = "2";
FileUtil.fullyDelete(new File(output));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
index c4920db..f0de098 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
@@ -37,13 +37,8 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * @author George Song (ysong1)
- *
- */
public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
@Before
public void setUp() throws Exception {
@@ -73,7 +68,7 @@ public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
+ byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 };
Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value));
mapReduceDriver.addInput(input1);
@@ -83,7 +78,7 @@ public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
assertEquals(4, result.size());
byte[] resultKey = { 0, 0, 0, 0, 0, 0, 1, 127, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
+ byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 };
Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue));
assertTrue(result.contains(output1));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/resources/data/6d_cuboid/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/6d_cuboid/part-r-00000 b/job/src/test/resources/data/6d_cuboid/part-r-00000
deleted file mode 100644
index b41ba18..0000000
Binary files a/job/src/test/resources/data/6d_cuboid/part-r-00000 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/resources/data/8d_cuboid/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/8d_cuboid/part-r-00000 b/job/src/test/resources/data/8d_cuboid/part-r-00000
new file mode 100644
index 0000000..d277125
Binary files /dev/null and b/job/src/test/resources/data/8d_cuboid/part-r-00000 differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/resources/data/base_cuboid/part-r-00000
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/base_cuboid/part-r-00000 b/job/src/test/resources/data/base_cuboid/part-r-00000
index 1a011a4..ed53ffb 100644
Binary files a/job/src/test/resources/data/base_cuboid/part-r-00000 and b/job/src/test/resources/data/base_cuboid/part-r-00000 differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/34f37260/job/src/test/resources/data/flat_table/000000_0
----------------------------------------------------------------------
diff --git a/job/src/test/resources/data/flat_table/000000_0 b/job/src/test/resources/data/flat_table/000000_0
index 058c92a..8b1b7cc 100644
Binary files a/job/src/test/resources/data/flat_table/000000_0 and b/job/src/test/resources/data/flat_table/000000_0 differ